def test_unsqueeze2():
n_in, n_out = 11, 13
n_batch, n_time = 3, 7
def model_func(wrapped_import, inputs: torch.Tensor):
return inputs.unsqueeze(3)
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (n_batch, n_in, n_time)).astype("float32")
verify_torch_and_convert_to_returnn(model_func, inputs=x)
def test_mnist():
def model_func(wrapped_import, inputs):
if wrapped_import:
torch = wrapped_import("torch")
nn = wrapped_import("torch.nn")
F = wrapped_import("torch.nn.functional")
else:
import torch
import torch.nn as nn
import torch.nn.functional as F
# directly from here: https://github.com/pytorch/examples/blob/master/mnist/main.py
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(1, 32, 3, 1)
self.conv2 = nn.Conv2d(32, 64, 3, 1)
self.dropout1 = nn.Dropout(0.25)
self.dropout2 = nn.Dropout(0.5)
self.fc1 = nn.Linear(9216, 128)
self.fc2 = nn.Linear(128, 10)
def forward(self, x):
x = self.conv1(x)
x = F.relu(x)
x = self.conv2(x)
x = F.relu(x)
x = F.max_pool2d(x, 2)
x = self.dropout1(x)
x = torch.flatten(x, 1)
x = self.fc1(x)
x = F.relu(x)
x = self.dropout2(x)
x = self.fc2(x)
output = F.log_softmax(x, dim=1)
return output
net = Net()
net = net.eval() # disable dropout
return net(inputs)
rnd = numpy.random.RandomState(42)
N, C, H, W = 64, 1, 28, 28
x = rnd.normal(0., 1., (N, C, H, W)).astype("float32")
verify_torch_and_convert_to_returnn(
model_func, inputs=x, inputs_data_kwargs={"shape": (C, H, W)})
def test_movedim():
n_in, n_out = 11, 13
n_batch, n_time = 3, 7
def model_func(wrapped_import, inputs: torch.Tensor):
if typing.TYPE_CHECKING or not wrapped_import:
import torch
else:
torch = wrapped_import("torch")
x = inputs # (B,F,T)
x = torch.nn.Conv1d(n_in, n_out, 3)(
x) # make it (B,T,F) in RETURNN. (B,F,T) in Torch.
x = torch.movedim(x, 0,
1) # stay (B,T,F) in RETURNN. (F,B,T) in Torch.
return x
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (n_batch, n_in, n_time)).astype("float32")
verify_torch_and_convert_to_returnn(model_func, inputs=x)
def test_conv2d():
n_in, n_out = 11, 13
n_batch, n_time1, n_time2 = 3, 17, 19
def model_func(wrapped_import, inputs: torch.Tensor):
if typing.TYPE_CHECKING or not wrapped_import:
import torch
else:
torch = wrapped_import("torch")
# {'class': 'transposed_conv', 'from': 'layer2', 'activation': None, 'with_bias': True,
# 'n_out': 192, 'filter_size': (10,), 'strides': (5,), 'remove_padding': (3,), 'output_padding': (1,)}
model = torch.nn.Conv2d(in_channels=n_in,
out_channels=n_out,
kernel_size=(3, 5),
stride=2)
return model(inputs)
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (n_batch, n_in, n_time1, n_time2)).astype("float32")
verify_torch_and_convert_to_returnn(model_func, inputs=x)
def test_functional_conv_transposed():
n_in, n_out = 11, 13
n_batch, n_time = 3, 7
kernel_size = 3
def model_func(wrapped_import, inputs: torch.Tensor):
if typing.TYPE_CHECKING or not wrapped_import:
import torch
import torch.nn.functional as F
else:
torch = wrapped_import("torch")
F = wrapped_import("torch.nn.functional")
rnd = numpy.random.RandomState(42)
weight = rnd.normal(0., 1.,
(n_in, n_out, kernel_size)).astype("float32")
weight = torch.from_numpy(weight)
return F.conv_transpose1d(inputs, weight=weight, stride=2)
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (n_batch, n_in, n_time)).astype("float32")
verify_torch_and_convert_to_returnn(model_func, inputs=x)
def test_custom_layer_norm():
N, F, T = 64, 11, 28
def model_func(wrapped_import, inputs):
if wrapped_import:
torch = wrapped_import("torch")
else:
import torch
class LayerNorm(torch.nn.LayerNorm):
def __init__(self, nout, dim=-1):
"""Construct an LayerNorm object."""
super(LayerNorm, self).__init__(nout, eps=1e-12)
self.dim = dim
def forward(self, x):
if self.dim == -1:
return super(LayerNorm, self).forward(x)
return super(LayerNorm,
self).forward(x.transpose(1,
-1)).transpose(1, -1)
class Net(torch.nn.Module):
def __init__(self):
super(Net, self).__init__()
self.norm = LayerNorm(nout=F, dim=1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.norm(x)
net = Net()
return net(inputs)
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (N, F, T)).astype("float32")
verify_torch_and_convert_to_returnn(model_func, inputs=x)
def test_flatten_batch():
n_in, n_out = 11, 13
n_batch, n_time = 3, 7
def model_func(wrapped_import, inputs: torch.Tensor):
if typing.TYPE_CHECKING or not wrapped_import:
import torch
else:
torch = wrapped_import("torch")
x = inputs # (B,F,T)
x = torch.movedim(x, 1, 0) # (F,B,T)
x = torch.reshape(x, (x.shape[0], -1)) # (F,B*T)
return x
rnd = numpy.random.RandomState(42)
x = rnd.normal(0., 1., (n_batch, n_in, n_time)).astype("float32")
converter = verify_torch_and_convert_to_returnn(model_func, inputs=x)
assert converter.returnn_net_dict["Flatten"][
"class"] == "flatten_batch", pformat(converter.returnn_net_dict)
def main():
parser = argparse.ArgumentParser(description="MB-MelGAN vocoder")
parser.add_argument("--features",
required=True,
help="npy file. via decoder.py --dump_features")
parser.add_argument("--pwg_config",
type=str,
help="ParallelWaveGAN config (.yaml)")
parser.add_argument("--pwg_checkpoint",
type=str,
help="ParallelWaveGAN checkpoint (.pkl)")
args = parser.parse_args()
better_exchook.install()
debug_register_better_repr()
log.initialize(verbosity=[5])
setup_tf_thread_pools()
print_available_devices()
def model_func(wrapped_import, inputs: torch.Tensor):
if typing.TYPE_CHECKING or not wrapped_import:
import torch
from parallel_wavegan import models as pwg_models
from parallel_wavegan import layers as pwg_layers
else:
torch = wrapped_import("torch")
wrapped_import("parallel_wavegan")
pwg_models = wrapped_import("parallel_wavegan.models")
pwg_layers = wrapped_import("parallel_wavegan.layers")
# Initialize PWG
pwg_config = yaml.load(open(args.pwg_config), Loader=yaml.Loader)
pyt_device = torch.device("cpu")
generator = pwg_models.MelGANGenerator(
**pwg_config['generator_params'])
generator.load_state_dict(
torch.load(args.pwg_checkpoint,
map_location="cpu")["model"]["generator"])
generator.remove_weight_norm()
pwg_model = generator.eval().to(pyt_device)
assert pwg_config["generator_params"].get(
"aux_context_window", 0) == 0 # not implemented otherwise
pwg_pqmf = pwg_layers.PQMF(
pwg_config["generator_params"]["out_channels"]).to(pyt_device)
with torch.no_grad():
return pwg_pqmf.synthesis(pwg_model(inputs))
feature_data = numpy.load(args.features)
print("Feature shape:", feature_data.shape)
import pytorch_to_returnn.log
pytorch_to_returnn.log.Verbosity = 6
from pytorch_to_returnn.converter import verify_torch_and_convert_to_returnn
verify_torch_and_convert_to_returnn(model_func,
inputs=feature_data[None, :, :])
# from pytorch_to_returnn.wrapped_import import wrapped_import_demo
# from pytorch_to_returnn import torch as torch_returnn
# model_func(wrapped_import_demo, inputs=torch_returnn.from_numpy(feature_data[None, :, :]))
audio_waveform = model_func(None,
inputs=torch.from_numpy(
feature_data[None, :, :]))
audio_waveform = audio_waveform.view(-1).cpu().numpy()
audio_raw = numpy.asarray(audio_waveform * (2**15 - 1),
dtype="int16").tobytes()
out_fn = "out.wav"
wave_writer = wave.open(out_fn, "wb")
wave_writer.setnchannels(1)
wave_writer.setframerate(16000)
wave_writer.setsampwidth(2)
wave_writer.writeframes(audio_raw)
wave_writer.close()
print("Wrote %s." % out_fn)
def main():
parser = argparse.ArgumentParser(
description='PyTorch Wikitext-2 RNN/LSTM/GRU/Transformer Language Model'
)
parser.add_argument('--data',
type=str,
default='./data/wikitext-2',
help='location of the data corpus')
parser.add_argument(
'--model',
type=str,
default='LSTM',
help=
'type of recurrent net (RNN_TANH, RNN_RELU, LSTM, GRU, Transformer)')
parser.add_argument('--emsize',
type=int,
default=200,
help='size of word embeddings')
parser.add_argument('--nhid',
type=int,
default=200,
help='number of hidden units per layer')
parser.add_argument('--nlayers',
type=int,
default=2,
help='number of layers')
parser.add_argument('--lr',
type=float,
default=20,
help='initial learning rate')
parser.add_argument('--clip',
type=float,
default=0.25,
help='gradient clipping')
parser.add_argument('--epochs',
type=int,
default=40,
help='upper epoch limit')
parser.add_argument('--batch_size',
type=int,
default=20,
metavar='N',
help='batch size')
parser.add_argument('--bptt', type=int, default=35, help='sequence length')
parser.add_argument('--dropout',
type=float,
default=0.2,
help='dropout applied to layers (0 = no dropout)')
parser.add_argument('--tied',
action='store_true',
help='tie the word embedding and softmax weights')
parser.add_argument('--seed', type=int, default=1111, help='random seed')
parser.add_argument('--cuda', action='store_true', help='use CUDA')
parser.add_argument('--log-interval',
type=int,
default=200,
metavar='N',
help='report interval')
parser.add_argument('--save',
type=str,
default='model.pt',
help='path to save the final model')
parser.add_argument('--onnx-export',
type=str,
default='',
help='path to export the final model in onnx format')
parser.add_argument(
'--nhead',
type=int,
default=2,
help=
'the number of heads in the encoder/decoder of the transformer model')
parser.add_argument('--dry-run',
action='store_true',
help='verify the code and the model')
args = parser.parse_args()
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print(
"WARNING: You have a CUDA device, so you should probably run with --cuda"
)
device = torch.device("cuda" if args.cuda else "cpu")
###############################################################################
# Load data
###############################################################################
corpus = data.Corpus(args.data)
# Starting from sequential data, batchify arranges the dataset into columns.
# For instance, with the alphabet as the sequence and batch size 4, we'd get
# ┌ a g m s ┐
# │ b h n t │
# │ c i o u │
# │ d j p v │
# │ e k q w │
# └ f l r x ┘.
# These columns are treated as independent by the model, which means that the
# dependence of e. g. 'g' on 'f' can not be learned, but allows more efficient
# batch processing.
def batchify(data, bsz):
# Work out how cleanly we can divide the dataset into bsz parts.
nbatch = data.size(0) // bsz
# Trim off any extra elements that wouldn't cleanly fit (remainders).
data = data.narrow(0, 0, nbatch * bsz)
# Evenly divide the data across the bsz batches.
data = data.view(bsz, -1).t().contiguous()
return data.to(device)
eval_batch_size = 10
train_data = batchify(corpus.train, args.batch_size)
val_data = batchify(corpus.valid, eval_batch_size)
test_data = batchify(corpus.test, eval_batch_size)
def get_batch(source, i):
seq_len = min(args.bptt, len(source) - 1 - i)
data = source[i:i + seq_len]
target = source[i + 1:i + 1 + seq_len].view(-1)
return data, target
ntokens = len(corpus.dictionary)
def model_func(wrapped_import, inputs):
###############################################################################
# Build the model
###############################################################################
if wrapped_import:
nn = wrapped_import("torch.nn")
model = wrapped_import("model")
else:
from torch import nn
import model
if args.model == 'Transformer':
net = model.TransformerModel(ntokens, args.emsize, args.nhead,
args.nhid, args.nlayers, args.dropout)
else:
net = model.RNNModel(args.model, ntokens, args.emsize, args.nhid,
args.nlayers, args.dropout, args.tied)
net.eval() # for verification, need no random elements (e.g. dropout)
# criterion = nn.NLLLoss()
if args.model != 'Transformer':
hidden = net.init_hidden(args.batch_size)
else:
hidden = None
with torch.no_grad():
if args.model == 'Transformer':
output = net(inputs)
output = output.view(-1, ntokens)
else:
output, hidden = net(inputs, hidden)
return output
data_, targets = get_batch(train_data, i=0)
from pytorch_to_returnn.converter import verify_torch_and_convert_to_returnn
verify_torch_and_convert_to_returnn(model_func,
inputs=data_.detach().cpu().numpy(),
inputs_data_kwargs={
"shape": (None, ),
"sparse": True,
"dim": ntokens
})
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=14,
metavar='N',
help='number of epochs to train (default: 14)')
parser.add_argument('--lr',
type=float,
default=1.0,
metavar='LR',
help='learning rate (default: 1.0)')
parser.add_argument('--gamma',
type=float,
default=0.7,
metavar='M',
help='Learning rate step gamma (default: 0.7)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--dry-run',
action='store_true',
default=False,
help='quickly check a single pass')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
train_kwargs = {'batch_size': args.batch_size}
test_kwargs = {'batch_size': args.test_batch_size}
if use_cuda:
cuda_kwargs = {'num_workers': 1, 'pin_memory': True, 'shuffle': True}
train_kwargs.update(cuda_kwargs)
test_kwargs.update(cuda_kwargs)
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
dataset1 = datasets.MNIST('../data',
train=True,
download=True,
transform=transform)
dataset2 = datasets.MNIST('../data', train=False, transform=transform)
train_loader = torch.utils.data.DataLoader(dataset1, **train_kwargs)
test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)
data, target = next(iter(train_loader))
data_np = data.numpy()
verify_torch_and_convert_to_returnn(
model_func, inputs=data_np, inputs_data_kwargs={"shape": (1, 28, 28)})
