def setUp(self):
self._params_1 = OrderedDict(
[
("double_tensor", torch.DoubleTensor([100.0])),
("float_tensor", torch.FloatTensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])),
("long_tensor", torch.LongTensor([7, 8, 9])),
("half_tensor", torch.HalfTensor([10.0, 20.0])),
]
)
self._params_2 = OrderedDict(
[
("double_tensor", torch.DoubleTensor([1.0])),
("float_tensor", torch.FloatTensor([[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]])),
# Any integer tensor must remain the same in all the params.
("long_tensor", torch.LongTensor([7, 8, 9])),
("half_tensor", torch.HalfTensor([50.0, 0.0])),
]
)
self._params_avg = OrderedDict(
[
("double_tensor", torch.DoubleTensor([50.5])),
("float_tensor", torch.FloatTensor([[1.0, 1.5, 2.0], [2.5, 3.0, 3.5]])),
("long_tensor", torch.LongTensor([7, 8, 9])),
# We convert fp16 to fp32 when averaging params.
("half_tensor", torch.FloatTensor([30.0, 10.0])),
]
)
self._fd_1, self._filename_1 = tempfile.mkstemp()
self._fd_2, self._filename_2 = tempfile.mkstemp()
torch.save(OrderedDict([("model", self._params_1)]), self._filename_1)
torch.save(OrderedDict([("model", self._params_2)]), self._filename_2)
def load_databatch(data_folder, idx, img_size=32):
data_file = os.path.join(data_folder, 'train_data_batch_')
d = unpickle(data_file + str(idx))
x = d['data']
y = d['labels']
mean_image = d['mean']
x = x / np.float16(255)
mean_image = mean_image / np.float16(255)
# Labels are indexed from 1, shift it so that indexes start at 0
y = [i - 1 for i in y]
data_size = x.shape[0]
x -= mean_image
img_size2 = img_size * img_size
x = np.dstack((x[:, :img_size2], x[:, img_size2:2 * img_size2], x[:, 2 * img_size2:]))
x = x.reshape((x.shape[0], img_size, img_size, 3)).transpose(0, 3, 1, 2)
# create mirrored images
X_train = x[0:data_size, :, :, :]
Y_train = y[0:data_size]
X_train_flip = X_train[:, :, :, ::-1]
Y_train_flip = Y_train
X_train = np.concatenate((X_train, X_train_flip), axis=0)
Y_train = np.concatenate((Y_train, Y_train_flip), axis=0)
return dict(
X_train=torch.HalfTensor(X_train),
Y_train=torch.LongTensor(Y_train),
mean=torch.HalfTensor(mean_image))
def infer(self, spect, sigma=1.0):
spect = self.upsample(spect)
# trim conv artifacts. maybe pad spec to kernel multiple
time_cutoff = self.upsample.kernel_size[0] - self.upsample.stride[0]
spect = spect[:, :, :-time_cutoff]
spect = spect.unfold(2, self.n_group, self.n_group).permute(0, 2, 1, 3)
spect = spect.contiguous().view(spect.size(0), spect.size(1), -1).permute(0, 2, 1)
# if spect.type() == 'torch.cuda.HalfTensor':
# audio = torch.cuda.HalfTensor(spect.size(0),
# self.n_remaining_channels,
# spect.size(2)).normal_()
# else:
# audio = torch.cuda.FloatTensor(spect.size(0),
# self.n_remaining_channels,
# spect.size(2)).normal_()
if spect.type() == 'torch.HalfTensor':
audio = torch.HalfTensor(spect.size(0),
self.n_remaining_channels,
spect.size(2)).normal_()
else:
audio = torch.FloatTensor(spect.size(0),
self.n_remaining_channels,
spect.size(2)).normal_()
audio = torch.autograd.Variable(sigma*audio)
for k in reversed(range(self.n_flows)):
n_half = int(audio.size(1)/2)
audio_0 = audio[:, :n_half, :]
audio_1 = audio[:, n_half:, :]
output = self.WN[k]((audio_0, spect))
s = output[:, n_half:, :]
b = output[:, :n_half, :]
audio_1 = (audio_1 - b)/torch.exp(s)
audio = torch.cat([audio_0, audio_1], 1)
audio = self.convinv[k](audio, reverse=True)
if k % self.n_early_every == 0 and k > 0:
# if spect.type() == 'torch.cuda.HalfTensor':
# z = torch.cuda.HalfTensor(spect.size(
# 0), self.n_early_size, spect.size(2)).normal_()
# else:
# z = torch.cuda.FloatTensor(spect.size(
# 0), self.n_early_size, spect.size(2)).normal_()
if spect.type() == 'torch.HalfTensor':
z = torch.HalfTensor(spect.size(
0), self.n_early_size, spect.size(2)).normal_()
else:
z = torch.FloatTensor(spect.size(
0), self.n_early_size, spect.size(2)).normal_()
audio = torch.cat((sigma*z, audio), 1)
audio = audio.permute(0, 2, 1).contiguous().view(
audio.size(0), -1).data
return audio
def validate(model, criterion, valset, iteration, batch_size, n_gpus,
collate_fn, logger, distributed_run, rank):
"""Handles all the validation scoring and printing"""
model.eval()
with torch.no_grad():
val_sampler = DistributedSampler(valset) if distributed_run else None
val_loader = DataLoader(valset, sampler=val_sampler, num_workers=1,
shuffle=False, batch_size=batch_size,
pin_memory=False, collate_fn=collate_fn)
val_loss = 0.0
for i, batch in enumerate(val_loader):
x, y, embedding = model.parse_batch(batch)
embedding_tensor = Variable(torch.HalfTensor(embedding).cuda(),requires_grad=False)
y_pred = model(x, embedding_tensor)
loss = criterion(y_pred, y)
if distributed_run:
reduced_val_loss = reduce_tensor(loss.data, n_gpus).item()
else:
reduced_val_loss = loss.item()
val_loss += reduced_val_loss
val_loss = val_loss / (i + 1)
model.train()
if rank == 0:
print("Validation loss {}: {:9f} ".format(iteration, reduced_val_loss))
logger.log_validation(reduced_val_loss, model, y, y_pred, iteration)
def proj_simplex(v, s=1, device = "cuda:0"):
assert s > 0, "Radius s must be strictly positive (%d <= 0)" % s
batch_size = v.shape[0]
# check if we are already on the simplex
'''
#Not checking this as we are calling this from the previous function only
if v.sum(dim = (1,2,3)) == s and np.alltrue(v >= 0):
# best projection: itself!
return v
'''
# get the array of cumulative sums of a sorted (decreasing) copy of v
u = v.view(batch_size,1,-1)
n = u.shape[2]
u, indices = torch.sort(u, descending = True)
cssv = u.cumsum(dim = 2)
# get the number of > 0 components of the optimal solution
vec = u * torch.arange(1, n+1).float().to(device) #.half()
comp = (vec > (cssv - s)).half()
u = comp.cumsum(dim = 2)
w = (comp-1).cumsum(dim = 2)
u = u + w
rho = torch.argmax(u, dim = 2)
rho = rho.view(batch_size)
c = torch.HalfTensor([cssv[i,0,rho[i]] for i in range( cssv.shape[0]) ]).to(device)
c = c-s
# compute the Lagrange multiplier associated to the simplex constraint
theta = torch.div(c,(rho.half() + 1))
theta = theta.view(batch_size,1,1,1)
# compute the projection by thresholding v using theta
w = (v - theta.float()).clamp(min=0)
return w
def _graph_constant(g, value, dims, type, *args, **kwargs):
assert isinstance(value, numbers.Number)
assert type is not None
isscalar = False
if dims is None or dims == 0 or set(dims) == set([0]):
dims = [1]
isscalar = True
type = type.lower()
if type == "char":
tensor = torch.CharTensor(*dims)
elif type == "short":
tensor = torch.ShortTensor(*dims)
elif type == "int":
tensor = torch.IntTensor(*dims)
elif type == "long":
tensor = torch.LongTensor(*dims)
elif type == "half":
tensor = torch.HalfTensor(*dims)
elif type == "float":
tensor = torch.FloatTensor(*dims)
elif type == "double":
tensor = torch.DoubleTensor(*dims)
else:
raise ValueError(
"Unknown type, type should be one of the following strings: "
"char, short, int, long, half, float, double")
tensor.fill_(value)
if isscalar:
return g.op("Constant", *args, value_z=tensor, **kwargs)
return g.op("Constant", *args, value_t=tensor, **kwargs)
def conv4d(data, filters, bias=None, permute_filters=True, use_half=False):
b, c, h, w, d, t = data.size()
# permute to avoid making contiguous inside loop
data = data.permute(2, 0, 1, 3, 4, 5).contiguous()
# Same permutation is done with filters, unless already provided with permutation
if permute_filters:
# permute to avoid making contiguous inside loop
filters = filters.permute(2, 0, 1, 3, 4, 5).contiguous()
c_out = filters.size(1)
if use_half:
output = Variable(torch.HalfTensor(h, b, c_out, w, d, t),
requires_grad=data.requires_grad)
else:
output = Variable(torch.zeros(h, b, c_out, w, d, t),
requires_grad=data.requires_grad)
padding = filters.size(0) // 2
if use_half:
Z = Variable(torch.zeros(padding, b, c, w, d, t).half())
else:
Z = Variable(torch.zeros(padding, b, c, w, d, t))
if data.is_cuda:
Z = Z.cuda(data.get_device())
output = output.cuda(data.get_device())
data_padded = torch.cat((Z, data, Z), 0)
for i in range(output.size(0)): # loop on first feature dimension
# convolve with center channel of filter (at position=padding)
output[i, :, :, :, :, :] = F.conv3d(
data_padded[i + padding, :, :, :, :, :],
filters[padding, :, :, :, :, :],
bias=bias,
stride=1,
padding=padding,
)
# convolve with upper/lower channels of filter (at postions [:padding] [padding+1:])
for p in range(1, padding + 1):
output[i, :, :, :, :, :] = output[i, :, :, :, :, :] + F.conv3d(
data_padded[i + padding - p, :, :, :, :, :],
filters[padding - p, :, :, :, :, :],
bias=None,
stride=1,
padding=padding,
)
output[i, :, :, :, :, :] = output[i, :, :, :, :, :] + F.conv3d(
data_padded[i + padding + p, :, :, :, :, :],
filters[padding + p, :, :, :, :, :],
bias=None,
stride=1,
padding=padding,
)
output = output.permute(1, 2, 0, 3, 4, 5).contiguous()
return output
def forward(self, batch_H, text, is_train=True, batch_max_length=25):
"""
input:
batch_H : contextual_feature H = hidden state of encoder. [batch_size x num_steps x num_classes]
text : the text-index of each image. [batch_size x (max_length+1)]. +1 for [GO] token. text[:, 0] = [GO].
output: probability distribution at each step [batch_size x num_steps x num_classes]
"""
batch_size = batch_H.size(0)
num_steps = batch_max_length + 1 # +1 for [s] at end of sentence.
output_hiddens = torch.HalfTensor(batch_size, num_steps,
self.hidden_size).fill_(0).to(device)
hidden = (torch.HalfTensor(batch_size,
self.hidden_size).fill_(0).to(device),
torch.HalfTensor(batch_size,
self.hidden_size).fill_(0).to(device))
if is_train:
for i in range(num_steps):
# one-hot vectors for a i-th char. in a batch
char_onehots = self._char_to_onehot(
text[:, i], onehot_dim=self.num_classes)
# hidden : decoder's hidden s_{t-1}, batch_H : encoder's hidden H, char_onehots : one-hot(y_{t-1})
hidden, alpha = self.attention_cell(hidden, batch_H,
char_onehots)
output_hiddens[:, i, :] = hidden[
0] # LSTM hidden index (0: hidden, 1: Cell)
probs = self.generator(output_hiddens)
else:
targets = torch.LongTensor(batch_size).fill_(0).to(
device) # [GO] token
probs = torch.FloatTensor(batch_size, num_steps,
self.num_classes).fill_(0).to(device)
for i in range(num_steps):
char_onehots = self._char_to_onehot(
targets, onehot_dim=self.num_classes)
hidden, alpha = self.attention_cell(hidden, batch_H,
char_onehots)
probs_step = self.generator(hidden[0])
probs[:, i, :] = probs_step
_, next_input = probs_step.max(1)
targets = next_input
return probs.float() # batch_size x num_steps x num_classes
def store(self, state, action, reward, done, next_state):
state = torch.Tensor(state).to("cpu")
action = torch.HalfTensor(action).to("cpu")
reward = torch.Tensor([reward]).to("cpu")
done = torch.BoolTensor([done]).to("cpu")
next_state = torch.Tensor(next_state).to("cpu")
self.memory.add(state, action, reward, done, next_state)
def transfer_superbatch_to_gpu(self):
if not self.gpu:
raise TypeError('gpu support was set to False')
for k, v in self.cpu_superbatch.items():
try:
self.gpu_superbatch[k].copy_(torch.HalfTensor(v))
except:
raise Exception('Problem with', k)
def predict_transform_half(prediction,
inp_dim,
anchors,
num_classes,
CUDA=True):
''' predict transform half
'''
batch_size = prediction.size(0)
stride = inp_dim // prediction.size(2)
bbox_attrs = 5 + num_classes
num_anchors = len(anchors)
grid_size = inp_dim // stride
prediction = prediction.view(batch_size, bbox_attrs * num_anchors,
grid_size * grid_size)
prediction = prediction.transpose(1, 2).contiguous()
prediction = prediction.view(batch_size,
grid_size * grid_size * num_anchors,
bbox_attrs)
# Sigmoid the centre_X, centre_Y. and object confidencce
prediction[:, :, 0] = torch.sigmoid(prediction[:, :, 0])
prediction[:, :, 1] = torch.sigmoid(prediction[:, :, 1])
prediction[:, :, 4] = torch.sigmoid(prediction[:, :, 4])
# Add the center offsets
grid_len = np.arange(grid_size)
a, b = np.meshgrid(grid_len, grid_len)
x_offset = torch.FloatTensor(a).view(-1, 1)
y_offset = torch.FloatTensor(b).view(-1, 1)
if CUDA:
x_offset = x_offset.cuda().half()
y_offset = y_offset.cuda().half()
x_y_offset = (torch.cat((x_offset, y_offset),
1).repeat(1, num_anchors).view(-1, 2).unsqueeze(0))
prediction[:, :, :2] += x_y_offset
# log space transform height and the width
anchors = torch.HalfTensor(anchors)
if CUDA:
anchors = anchors.cuda()
anchors = anchors.repeat(grid_size * grid_size, 1).unsqueeze(0)
prediction[:, :, 2:4] = torch.exp(prediction[:, :, 2:4]) * anchors
# Softmax the class scores
prediction[:, :, 5:5 + num_classes] = nn.Softmax(-1)(Variable(
prediction[:, :, 5:5 + num_classes])).data
prediction[:, :, :4] *= stride
return prediction
def collate_src(self, insts):
max_len = max(inst.shape[0] for inst in insts)
inputs = np.zeros((len(insts), max_len, insts[0].shape[1]))
masks = torch.zeros((len(insts), max_len), dtype=torch.uint8)
for idx, inst in enumerate(insts):
inputs[idx, :inst.shape[0], :] = inst
masks[idx, :inst.shape[0]] = 1
inputs = torch.HalfTensor(inputs) if self.fp16 else torch.FloatTensor(inputs)
return inputs, masks
def collate_src_pack(self, insts):
max_len = max(inst.shape[0] for inst in insts)
masks = torch.zeros((len(insts), max_len), dtype=torch.uint8)
inputs = []
for idx, inst in enumerate(insts):
inputs.append(torch.HalfTensor(inst) if self.fp16 else torch.FloatTensor(inst))
masks[idx, 0:inst.shape[0]] = 1
inputs = pack_sequence(inputs)
return inputs, masks
def __init__(self, opt, use_cuda=True, output_bgr=True, debug=False):
self.use_cuda = use_cuda
self.output_bgr = output_bgr
self.debug = debug
# for k,v in opt._get_kwargs():
# print(k,v)
# Load model params
self.model = translation_generator.get_net(opt)
self.model.load_state_dict(
torch.load(opt.checkpoint_path, map_location=torch.device('cpu')))
self.model.eval()
if self.use_cuda: self.model.cuda()
# Load textures
self.texture = texture_model.get_net(opt)
self.texture.load_state_dict(
torch.load(opt.checkpoint_path_texture,
map_location=torch.device('cpu')))
self.texture.eval()
if self.use_cuda: self.texture.cuda()
if opt.is_half:
print('[ImageRenderer]: FP16 mode')
if self.use_cuda:
self.to_tensor = lambda x: torch.HalfTensor(x).cuda()
else:
self.to_tensor = lambda x: torch.HalfTensor(x)
self.model.half()
self.texture.half()
else:
if self.use_cuda:
self.to_tensor = lambda x: torch.FloatTensor(x).cuda()
else:
self.to_tensor = lambda x: torch.FloatTensor(x)
# Load stickman drawer
self.stickman_drawer = st.StickmanData_C(False)
self.opt = opt
def __getitem__(self, index: int) -> torch.Tensor:
"""Return a partition of the song's spectrogram, its index, and the ground truth sequence."""
name, frame = self.index_to_context[index]
tensor, indices = self.features[name]
targets = self.targets[name]
#tensor, indices = torch.load(f'{EXTRACT_PATH}/{name}/features.pt')
#targets = torch.load(f'{EXTRACT_PATH}/{name}/targets.pt')
context = tensor.shape[-1] // len(indices)
start = frame * context
end = (frame + 1) * context
frame = torch.HalfTensor([frame])
return ((tensor[:, :, start:end], frame), targets[start:end])
def test_all_dtypes():
return (
torch.BoolTensor([2]),
torch.LongTensor([3]),
torch.ByteTensor([4]),
torch.CharTensor([5]),
torch.DoubleTensor([6]),
torch.FloatTensor([7]),
torch.IntTensor([8]),
torch.ShortTensor([1]),
torch.HalfTensor([1]),
)
def __init__(self, file_path, file_doclens):
self.dim = 128 # TODO
self.doclens = file_doclens
self.endpos = np.cumsum(self.doclens)
self.startpos = self.endpos - self.doclens
mmap_storage = torch.HalfStorage.from_file(
file_path, False,
sum(self.doclens) * self.dim)
self.mmap = torch.HalfTensor(mmap_storage).view(
sum(self.doclens), self.dim)
def predict_transform_half(prediction,
inp_dim,
anchors,
num_classes,
CUDA=True):
batch_size = prediction.size(0)
stride = inp_dim // prediction.size(2)
bbox_attrs = 5 + num_classes
num_anchors = len(anchors)
grid_size = inp_dim // stride
prediction = prediction.view(batch_size, bbox_attrs * num_anchors,
grid_size * grid_size)
prediction = prediction.transpose(1, 2).contiguous()
prediction = prediction.view(batch_size,
grid_size * grid_size * num_anchors,
bbox_attrs)
prediction[:, :, 0] = torch.sigmoid(prediction[:, :, 0])
prediction[:, :, 1] = torch.sigmoid(prediction[:, :, 1])
prediction[:, :, 4] = torch.sigmoid(prediction[:, :, 4])
grid_len = np.arange(grid_size)
a, b = np.meshgrid(grid_len, grid_len)
x_offset = torch.FloatTensor(a).view(-1, 1)
y_offset = torch.FloatTensor(b).view(-1, 1)
if CUDA:
x_offset = x_offset.cuda().half()
y_offset = y_offset.cuda().half()
x_y_offset = torch.cat((x_offset, y_offset),
1).repeat(1, num_anchors).view(-1, 2).unsqueeze(0)
prediction[:, :, :2] += x_y_offset
anchors = torch.HalfTensor(anchors)
if CUDA:
anchors = anchors.cuda()
anchors = anchors.repeat(grid_size * grid_size, 1).unsqueeze(0)
prediction[:, :, 2:4] = torch.exp(prediction[:, :, 2:4]) * anchors
prediction[:, :, 5:5 + num_classes] = nn.Softmax(-1)(Variable(
prediction[:, :, 5:5 + num_classes])).data
prediction[:, :, :4] *= stride
return prediction
def _char_to_onehot(self, input_char, onehot_dim=38):
input_char = input_char.unsqueeze(1)
batch_size = input_char.size(0)
if self.opt.amp > 0:
one_hot = torch.HalfTensor(batch_size,
onehot_dim).zero_().to(self.opt.device)
else:
one_hot = torch.FloatTensor(batch_size,
onehot_dim).zero_().to(self.opt.device)
one_hot = one_hot.scatter_(1, input_char, 1)
return one_hot
def predict_transform_half(prediction,
inp_dim,
anchors,
num_classes,
use_cuda=True):
batch_size = prediction.size(0)
stride = inp_dim // prediction.size(2)
bbox_attrs = 5 + num_classes
num_anchors = len(anchors)
grid_size = inp_dim // stride
prediction = prediction.view(batch_size, bbox_attrs * num_anchors,
grid_size * grid_size)
prediction = prediction.transpose(1, 2).contiguous()
prediction = prediction.view(batch_size,
grid_size * grid_size * num_anchors,
bbox_attrs)
# Apply sigmoid to the bounding box center x, center y, and objectness
prediction[:, :, 0] = torch.sigmoid(prediction[:, :, 0])
prediction[:, :, 1] = torch.sigmoid(prediction[:, :, 1])
prediction[:, :, 4] = torch.sigmoid(prediction[:, :, 4])
# Add in the appropriate offsets for the bounding box center
grid_len = np.arange(grid_size)
a, b = np.meshgrid(grid_len, grid_len)
x_offset = torch.FloatTensor(a).view(-1, 1)
y_offset = torch.FloatTensor(b).view(-1, 1)
if use_cuda:
x_offset = x_offset.cuda().half()
y_offset = y_offset.cuda().half()
x_y_offset = torch.cat((x_offset, y_offset),
1).repeat(1, num_anchors).view(-1, 2).unsqueeze(0)
prediction[:, :, :2] += x_y_offset
# Width and height are relative to the anchors
anchors = torch.HalfTensor(anchors)
if use_cuda:
anchors = anchors.cuda()
anchors = anchors.repeat(grid_size * grid_size, 1).unsqueeze(0)
prediction[:, :, 2:4] = torch.exp(prediction[:, :, 2:4]) * anchors
# Class scores need sigmoid activation
prediction[:, :, 5:(5 + num_classes)] = nn.Softmax(-1)(Variable(
prediction[:, :, 5:(5 + num_classes)])).data
prediction[:, :, :4] *= stride
return prediction
def _graph_constant(
g,
value,
dims,
type_: str,
*args,
**kwargs,
):
"""This helper function can create either constant tensor or constant scalar.
If dims is None or 0 or [0], generate a 0-d tensor (scalar).
"""
assert isinstance(value, numbers.Number)
assert type_ is not None
isscalar = False
if dims is None or dims == 0 or set(dims) == {0}:
dims = [1]
isscalar = True
type_ = type_.lower()
tensor: Union[
torch.CharTensor,
torch.ShortTensor,
torch.IntTensor,
torch.LongTensor,
torch.HalfTensor,
torch.FloatTensor,
torch.DoubleTensor,
]
if type_ == "char":
tensor = torch.CharTensor(*dims)
elif type_ == "short":
tensor = torch.ShortTensor(*dims)
elif type_ == "int":
tensor = torch.IntTensor(*dims)
elif type_ == "long":
tensor = torch.LongTensor(*dims)
elif type_ == "half":
tensor = torch.HalfTensor(*dims)
elif type_ == "float":
tensor = torch.FloatTensor(*dims)
elif type_ == "double":
tensor = torch.DoubleTensor(*dims)
else:
raise ValueError(
"Unknown type, type should be one of the following strings: "
"char, short, int, long, half, float, double"
)
tensor.fill_(value) # type: ignore[call-overload]
if isscalar:
return g.op("Constant", *args, value_z=tensor, **kwargs)
return g.op("Constant", *args, value_t=tensor, **kwargs)
def T(a, half=False, cuda=True):
"""
Convert numpy array into a pytorch tensor.
if Cuda is available and USE_GPU=True, store resulting tensor in GPU.
"""
if not torch.is_tensor(a):
a = np.array(np.ascontiguousarray(a))
if a.dtype in (np.int8, np.int16, np.int32, np.int64):
a = torch.LongTensor(a.astype(np.int64))
elif a.dtype in (np.float32, np.float64):
a = torch.HalfTensor(a) if half else torch.FloatTensor(a)
else: raise NotImplementedError(a.dtype)
if cuda: a = to_gpu(a, async=True)
return a
def infer(self, spect, sigma=1.0):
spect_size = spect.size()
l = spect.size(2) * (256 // self.n_audio_channel)
spect = spect.to(torch.float32)
if spect.type() == 'torch.HalfTensor':
audio = torch.HalfTensor(spect.size(0), self.n_remaining_channels,
l).normal_()
else:
audio = torch.FloatTensor(spect.size(0), self.n_remaining_channels,
l).normal_()
for k in reversed(range(self.n_flows)):
n_half = int(audio.size(1) / 2)
audio_0 = audio[:, :n_half, :]
audio_1 = audio[:, n_half:, :]
output = self.WN[k]((audio_0, spect))
s = output[:, n_half:, :]
b = output[:, :n_half, :]
audio_1 = (audio_1 - b) / torch.exp(s)
audio = torch.cat([audio_0, audio_1], 1)
audio = self.convinv[k](audio, reverse=True)
if k % self.n_early_every == 0 and k > 0:
if spect.type() == 'torch.HalfTensor':
z = torch.HalfTensor(spect.size(0), self.n_early_size,
l).normal_()
else:
z = torch.FloatTensor(spect.size(0), self.n_early_size,
l).normal_()
audio = torch.cat((sigma * z, audio), 1)
audio = audio.permute(0, 2, 1).contiguous().view(audio.size(0),
-1).data
return audio
def transfer(tag, send_buf, shape, half=False):
if shape == None:
left, right = get_left_right(tag)
send_opt = dist.isend(tensor=send_buf, dst=right)
send_opt.wait()
return None
elif not torch.is_tensor(send_buf):
left, right = get_left_right(tag)
try:
if half:
recv_buf = torch.HalfTensor(torch.Size(shape))
else:
recv_buf = torch.zeros(shape,
dtype=torch.int8) #, dtype=torch.int8
dist.recv(tensor=recv_buf, src=left)
except RuntimeError as error:
print("runtime error..")
return None
return recv_buf
else:
left, right = get_left_right(tag)
send_opt = dist.isend(tensor=send_buf, dst=right)
try:
if half:
recv_buf = torch.HalfTensor(torch.Size(shape))
else:
recv_buf = torch.zeros(shape, dtype=torch.int8)
dist.recv(tensor=recv_buf, src=left)
except RuntimeError as error:
print("runtime error")
return None
send_opt.wait()
return recv_buf
def inference(self, src_wav: np.ndarray, source_speaker: str,
target_speaker: str) -> Tensor:
"""Inference one utterance."""
with torch.no_grad():
src_wav = torch.HalfTensor(src_wav)
src_wav = self.preprocess(src_wav)
source_speaker_id = self.get_speaker_id(source_speaker,
src_wav.size(0))
target_speaker_id = self.get_speaker_id(target_speaker,
src_wav.size(0))
latent = self.model.forward(src_wav, source_speaker_id)[0]
result = self.model.reverse(latent, target_speaker_id)
result = result.cpu()
result *= self.window
return self.flatten(result)
def predict_transform_half(prediction,
inp_dim,
anchors,
num_classes,
CUDA=True):
batch_size = prediction.size(0)
stride = inp_dim // prediction.size(2) ##416//13=32
bbox_attrs = 5 + num_classes
num_anchors = len(anchors)
grid_size = inp_dim // stride # feature map每条边格子的数量，416//32=13
prediction = prediction.view(batch_size, bbox_attrs * num_anchors,
grid_size * grid_size)
prediction = prediction.transpose(1, 2).contiguous()
prediction = prediction.view(batch_size,
grid_size * grid_size * num_anchors,
bbox_attrs)
#Sigmoid the centre_X, centre_Y. and object confidencce
prediction[:, :, 0] = torch.sigmoid(prediction[:, :, 0])
prediction[:, :, 1] = torch.sigmoid(prediction[:, :, 1])
prediction[:, :, 4] = torch.sigmoid(prediction[:, :, 4])
#Add the center offsets
grid_len = np.arange(grid_size)
a, b = np.meshgrid(grid_len, grid_len)
x_offset = torch.FloatTensor(a).view(-1, 1)
y_offset = torch.FloatTensor(b).view(-1, 1)
if CUDA:
x_offset = x_offset.cuda().half()
y_offset = y_offset.cuda().half()
# 这里的x_y_offset对应的是最终的feature map中每个格子的左上角坐标，比如有13个格子，刚x_y_offset的坐标就对应为(0,0),(0,1)…(12,12) .view(-1, 2)将tensor变成两列，unsqueeze(0)在0维上添加了一维。
x_y_offset = torch.cat((x_offset, y_offset),
1).repeat(1, num_anchors).view(-1, 2).unsqueeze(0)
prediction[:, :, :2] += x_y_offset
#log space transform height and the width
anchors = torch.HalfTensor(anchors)
if CUDA:
anchors = anchors.cuda() #长度为6的list(三个anchors每个2个坐标)，
anchors = anchors.repeat(grid_size * grid_size, 1).unsqueeze(0)
prediction[:, :, 2:4] = torch.exp(prediction[:, :, 2:4]) * anchors
#Softmax the class scores
prediction[:, :, 5:5 + num_classes] = nn.Softmax(-1)(Variable(
prediction[:, :, 5:5 + num_classes])).data
prediction[:, :, :4] *= stride
return prediction
def forward(self, mel, text, mask, out_lens):
#dummy = torch.FloatTensor(1, mel.size(1), mel.size(2)).zero_()
dummy = torch.HalfTensor(1, mel.size(1),
mel.size(2)).zero_() ################### fp16
dummy = dummy.type(mel.type())
# seq_len x batch x dim
mel0 = torch.cat([dummy, mel[:-1, :, :]], 0)
if out_lens is not None:
# collect decreasing length indices
lens, ids = torch.sort(out_lens, descending=True)
original_ids = [0] * lens.size(0)
for i in range(len(ids)):
original_ids[ids[i]] = i
# mel_seq_len x batch x hidden_dim
attention_hidden = self.run_padded_sequence(
ids, original_ids, lens, mel0, self.attention_lstm)
else:
attention_hidden = self.attention_lstm(mel0)[0]
# attention weights batch x text_seq_len x mel_seq_len
# sums to 1 over text_seq_len
attention_context, attention_weights = self.attention_layer(
attention_hidden, text, text, mask=mask)
attention_context = attention_context.permute(2, 0, 1)
decoder_input = torch.cat((attention_hidden, attention_context), -1)
gates = None
if hasattr(self, 'gate_layer'):
# compute gates before packing
gates = self.gate_layer(decoder_input)
if out_lens is not None:
# reorder, run padded sequence and undo reordering
lstm_hidden = self.run_padded_sequence(ids, original_ids, lens,
decoder_input, self.lstm)
else:
lstm_hidden = self.lstm(decoder_input)[0]
lstm_hidden = self.dense_layer(lstm_hidden).permute(1, 2, 0)
decoder_output = self.conv(lstm_hidden).permute(2, 0, 1)
log_s = decoder_output[:, :, :mel.size(2)]
b = decoder_output[:, :, mel.size(2):]
mel = torch.exp(log_s) * mel + b
return mel, log_s, gates, attention_weights
def gaussian_kernel(self, h, w, center):
# Build a gaussian map
x, y = center
h, w, x, y = h // self.stride, w // self.stride, np.array(
x // self.stride), np.array(y // self.stride)
if x < 0 or y < 0:
ans = np.ones((h, w)) * -1
print(f'Image with -1')
else:
ycoords, xcoords = np.mgrid[0:h, 0:w]
num = -1 * (np.square(ycoords - y) + np.square(xcoords - x))
den = 2 * np.square(self.sigma)
ans = np.exp(num / den)
ans = ans / (np.sum(ans))
#ans = normalize(ans)
tensor_out = torch.HalfTensor(ans)
tensor_out.requires_grad = False
return tensor_out
def return_batch(self):
i1 = self.batch_index * self.batch_size
i2 = i1 + self.batch_size
if self.gpu == 1:
for k, v in self.cpu_superbatch.items():
try:
self.gpu_batch[k].copy_(torch.HalfTensor(v[i1:i2]))
except:
raise Exception('Problem with', k)
batch = self.gpu_batch
else:
if self.gpu == 0:
superbatch = self.cpu_superbatch
elif self.gpu == 2:
superbatch = self.gpu_superbatch
batch = {k: v[i1:i2] for k, v in superbatch.items()}
return batch
def proj_simplex(v, s=1):
assert s > 0, "Radius s must be strictly positive (%d <= 0)" % s
batch_size = v.shape[0]
u = v.view(batch_size, 1, -1)
n = u.shape[2]
u, indices = torch.sort(u, descending=True)
cssv = u.cumsum(dim=2)
vec = u * torch.arange(1, n + 1).float().cuda()
comp = (vec > (cssv - s)).half()
u = comp.cumsum(dim=2)
w = (comp - 1).cumsum(dim=2)
u = u + w
rho = torch.argmax(u, dim=2)
rho = rho.view(batch_size)
c = torch.HalfTensor([cssv[i, 0, rho[i]]
for i in range(cssv.shape[0])]).cuda()
c = c - s
theta = torch.div(c.float(), (rho.float() + 1))
theta = theta.view(batch_size, 1, 1, 1)
w = (v.float() - theta).clamp(min=0)
return w