def _str(self):
if self.is_sparse:
size_str = str(tuple(self.shape)).replace(' ', '')
return '{} of size {} with indices:\n{}\nand values:\n{}'.format(
self.type(), size_str, self._indices(), self._values())
prefix = 'tensor('
indent = len(prefix)
summarize = self.numel() > PRINT_OPTS.threshold
suffix = ')'
if not torch._C._is_default_type_cuda():
if self.device.type == 'cuda':
suffix = ', device=\'' + str(self.device) + '\'' + suffix
else:
if self.device.type == 'cpu' or torch.cuda.current_device() != self.device.index:
suffix = ', device=\'' + str(self.device) + '\'' + suffix
if self.numel() == 0:
# In an empty tensor, there are no elements to infer if the dtype should be int64,
# so it must be shown explicitly.
if self.dtype != torch.get_default_dtype():
suffix = ', dtype=' + str(self.dtype) + suffix
tensor_str = '[]'
else:
if self.dtype != torch.get_default_dtype() and self.dtype != torch.int64:
suffix = ', dtype=' + str(self.dtype) + suffix
fmt, scale, sz = _number_format(get_summarized_data(self) if summarize else self)
if scale != 1:
prefix = prefix + SCALE_FORMAT.format(scale) + ' ' * indent
tensor_str = _tensor_str(self, indent, fmt, scale, sz, summarize)
return prefix + tensor_str + suffix
def _str(self):
if self.is_sparse:
size_str = str(tuple(self.shape)).replace(' ', '')
return '{} of size {} with indices:\n{}\nand values:\n{}'.format(
self.type(), size_str, self._indices(), self._values())
prefix = 'tensor('
indent = len(prefix)
summarize = self.numel() > PRINT_OPTS.threshold
suffix = ''
if not torch._C._is_default_type_cuda():
if self.device.type == 'cuda':
suffix += ', device=\'' + str(self.device) + '\''
else:
if self.device.type == 'cpu' or torch.cuda.current_device() != self.device.index:
suffix += ', device=\'' + str(self.device) + '\''
if self.numel() == 0:
# Explicitly print the shape if it is not (0,), to match NumPy behavior
if self.dim() != 1:
suffix += ', size=' + str(tuple(self.shape))
# In an empty tensor, there are no elements to infer if the dtype should be int64,
# so it must be shown explicitly.
if self.dtype != torch.get_default_dtype():
suffix += ', dtype=' + str(self.dtype)
tensor_str = '[]'
else:
if self.dtype != torch.get_default_dtype() and self.dtype != torch.int64:
suffix += ', dtype=' + str(self.dtype)
formatter = _Formatter(get_summarized_data(self) if summarize else self)
tensor_str = _tensor_str(self, indent, formatter, summarize)
if self.grad_fn is not None:
suffix += ', grad_fn=<{}>'.format(type(self.grad_fn).__name__)
elif self.requires_grad:
suffix += ', requires_grad=True'
suffix += ')'
suffix = _maybe_wrap_suffix(suffix, indent, tensor_str)
return prefix + tensor_str + suffix
def forward(self, prob_map, gt, orig_sizes):
"""
Compute the Weighted Hausdorff Distance function
between the estimated probability map and ground truth points.
The output is the WHD averaged through all the batch.
:param prob_map: (B x H x W) Tensor of the probability map of the estimation.
B is batch size, H is height and W is width.
Values must be between 0 and 1.
:param gt: List of Tensors of the Ground Truth points.
Must be of size B as in prob_map.
Each element in the list must be a 2D Tensor,
where each row is the (y, x), i.e, (row, col) of a GT point.
:param orig_sizes: Bx2 Tensor containing the size
of the original images.
B is batch size.
The size must be in (height, width) format.
:param orig_widths: List of the original widths for each image
in the batch.
:return: Single-scalar Tensor with the Weighted Hausdorff Distance.
If self.return_2_terms=True, then return a tuple containing
the two terms of the Weighted Hausdorff Distance.
"""
_assert_no_grad(gt)
assert prob_map.dim() == 3, 'The probability map must be (B x H x W)'
assert prob_map.size()[1:3] == (self.height, self.width), \
'You must configure the WeightedHausdorffDistance with the height and width of the ' \
'probability map that you are using, got a probability map of size %s'\
% str(prob_map.size())
batch_size = prob_map.shape[0]
assert batch_size == len(gt)
terms_1 = []
terms_2 = []
for b in range(batch_size):
# One by one
prob_map_b = prob_map[b, :, :]
gt_b = gt[b]
orig_size_b = orig_sizes[b, :]
norm_factor = (orig_size_b / self.resized_size).unsqueeze(0)
n_gt_pts = gt_b.size()[0]
# Corner case: no GT points
if gt_b.ndimension() == 1 and (gt_b < 0).all().item() == 0:
terms_1.append(
torch.tensor([0], dtype=torch.get_default_dtype()))
terms_2.append(
torch.tensor([self.max_dist],
dtype=torch.get_default_dtype()))
continue
# Pairwise distances between all possible locations and the GTed locations
n_gt_pts = gt_b.size()[0]
normalized_x = norm_factor.repeat(self.n_pixels, 1) *\
self.all_img_locations
normalized_y = norm_factor.repeat(len(gt_b), 1) * gt_b
d_matrix = cdist(normalized_x, normalized_y)
# Reshape probability map as a long column vector,
# and prepare it for multiplication
p = prob_map_b.view(prob_map_b.nelement())
n_est_pts = p.sum()
p_replicated = p.view(-1, 1).repeat(1, n_gt_pts)
# Weighted Hausdorff Distance
term_1 = (1 / (n_est_pts + 1e-6)) * \
torch.sum(p * torch.min(d_matrix, 1)[0])
weighted_d_matrix = (
1 - p_replicated) * self.max_dist + p_replicated * d_matrix
minn = generaliz_mean(weighted_d_matrix,
p=self.p,
dim=0,
keepdim=False)
term_2 = torch.mean(minn)
# terms_1[b] = term_1
# terms_2[b] = term_2
terms_1.append(term_1)
terms_2.append(term_2)
terms_1 = torch.stack(terms_1)
terms_2 = torch.stack(terms_2)
if self.return_2_terms:
res = terms_1.mean(), terms_2.mean()
else:
res = terms_1.mean() + terms_2.mean()
return res
def __init__(
self,
num_embeddings: int,
embedding_dim: Optional[int] = None,
shape: Union[None, int, Sequence[int]] = None,
initializer: Hint[Initializer] = None,
initializer_kwargs: Optional[Mapping[str, Any]] = None,
normalizer: Hint[Normalizer] = None,
normalizer_kwargs: Optional[Mapping[str, Any]] = None,
constrainer: Hint[Constrainer] = None,
constrainer_kwargs: Optional[Mapping[str, Any]] = None,
regularizer: Hint[Regularizer] = None,
regularizer_kwargs: Optional[Mapping[str, Any]] = None,
trainable: bool = True,
dtype: Optional[torch.dtype] = None,
dropout: Optional[float] = None,
):
"""Instantiate an embedding with extended functionality.
:param num_embeddings: >0
The number of embeddings.
:param embedding_dim: >0
The embedding dimensionality.
:param initializer:
An optional initializer, which takes an uninitialized (num_embeddings, embedding_dim) tensor as input,
and returns an initialized tensor of same shape and dtype (which may be the same, i.e. the
initialization may be in-place). Can be passed as a function, or as string corresponding to a key in
:data:`pykeen.nn.emb.initializers` such as:
- ``"xavier_uniform"``
- ``"xavier_uniform_norm"``
- ``"xavier_normal"``
- ``"xavier_normal_norm"``
- ``"normal"``
- ``"normal_norm"``
- ``"uniform"``
- ``"uniform_norm"``
- ``"init_phases"``
:param initializer_kwargs:
Additional keyword arguments passed to the initializer
:param normalizer:
A normalization function, which is applied in every forward pass.
:param normalizer_kwargs:
Additional keyword arguments passed to the normalizer
:param constrainer:
A function which is applied to the weights after each parameter update, without tracking gradients.
It may be used to enforce model constraints outside of gradient-based training. The function does not need
to be in-place, but the weight tensor is modified in-place. Can be passed as a function, or as a string
corresponding to a key in :data:`pykeen.nn.emb.constrainers` such as:
- ``'normalize'``
- ``'complex_normalize'``
- ``'clamp'``
- ``'clamp_norm'``
:param constrainer_kwargs:
Additional keyword arguments passed to the constrainer
:param regularizer:
A regularizer, which is applied to the selected embeddings in forward pass
:param regularizer_kwargs:
Additional keyword arguments passed to the regularizer
:param dropout:
A dropout value for the embeddings.
"""
# normalize embedding_dim vs. shape
_embedding_dim, shape = process_shape(embedding_dim, shape)
if dtype is None:
dtype = torch.get_default_dtype()
# work-around until full complex support (torch==1.10 still does not work)
# TODO: verify that this is our understanding of complex!
if dtype.is_complex:
shape = tuple(shape[:-1]) + (2 * shape[-1], )
_embedding_dim = _embedding_dim * 2
# note: this seems to work, as finfo returns the datatype of the underlying floating
# point dtype, rather than the combined complex one
dtype = getattr(torch, torch.finfo(dtype).dtype)
super().__init__(
max_id=num_embeddings,
shape=shape,
)
# use make for initializer since there's a default, and make_safe
# for the others to pass through None values
self.initializer = initializer_resolver.make(initializer,
initializer_kwargs)
self.normalizer = normalizer_resolver.make_safe(
normalizer, normalizer_kwargs)
self.constrainer = constrainer_resolver.make_safe(
constrainer, constrainer_kwargs)
self.regularizer = regularizer_resolver.make_safe(
regularizer, regularizer_kwargs)
self._embeddings = torch.nn.Embedding(
num_embeddings=num_embeddings,
embedding_dim=_embedding_dim,
dtype=dtype,
)
self._embeddings.requires_grad_(trainable)
self.dropout = None if dropout is None else nn.Dropout(dropout)
def test_true_divide(self, device, dtype):
dividend = torch.randn(5, device=device).to(dtype)
divisor = torch.arange(1, 6, device=device).to(dtype)
casting_result = dividend.to(torch.get_default_dtype()) / divisor.to(torch.get_default_dtype())
self.assertEqual(casting_result, torch.true_divide(dividend, divisor))
def fdata(self,
dtype=None,
device=None,
rand=False,
missing=None,
cutoff=None,
dim=None,
numpy=False):
"""Load the scaled array in memory
This function differs from `data` in several ways:
* The output data type should be a floating point type.
* If an affine scaling (slope, intercept) is defined in the
file, it is applied to the data.
* the default output data type is `torch.get_default_dtype()`.
Parameters
----------
dtype : dtype_like, optional
Output data type. By default, use `torch.get_default_dtype()`.
Should be a floating point type.
device : torch.device, default='cpu'
Output device.
rand : bool, default=False
If the on-disk dtype is not floating point, sample noise
in the uncertainty interval.
missing : float or sequence[float], optional
Value(s) that correspond to missing values.
No noise is added to them, and they are converted to NaNs.
cutoff : float or (float, float), default=(0, 1)
Percentile cutoff. If only one value is provided, it is
assumed to relate to the upper percentile.
dim : int or list[int], optional
Dimensions along which to compute percentiles.
By default, they are computed on the flattened array.
numpy : bool, default=False
Return a numpy array rather than a torch tensor.
Returns
-------
dat : tensor[dtype]
"""
# --- sanity check ---
dtype = torch.get_default_dtype() if dtype is None else dtype
info = dtypes.dtype(dtype)
if not info.is_floating_point:
raise TypeError('Output data type should be a floating point '
'type but got {}.'.format(dtype))
# --- get unscaled data ---
dat = self.data(dtype=dtype,
device=device,
rand=rand,
missing=missing,
cutoff=cutoff,
dim=dim,
numpy=numpy)
# --- scale ---
if self.slope != 1:
dat *= float(self.slope)
if self.inter != 0:
dat += float(self.inter)
return dat
def _str_intern(inp, *, tensor_contents=None):
is_plain_tensor = type(inp) is torch.Tensor or type(
inp) is torch.nn.Parameter
if inp.is_nested:
prefix = "nested_tensor("
elif is_plain_tensor:
prefix = "tensor("
else:
prefix = f"{type(inp).__name__}("
indent = len(prefix)
suffixes = []
custom_contents_provided = tensor_contents is not None
if custom_contents_provided:
tensor_str = tensor_contents
# This is used to extract the primal value and thus disable the forward AD
# within this function.
# TODO(albanD) This needs to be updated when more than one level is supported
self, tangent = torch.autograd.forward_ad.unpack_dual(inp)
# Note [Print tensor device]:
# A general logic here is we only print device when it doesn't match
# the device specified in default tensor type.
# Currently torch.set_default_tensor_type() only supports CPU/CUDA, thus
# torch._C._get_default_device() only returns either cpu or cuda.
# In other cases, we don't have a way to set them as default yet,
# and we should always print out device for them.
if (self.device.type != torch._C._get_default_device()
or (self.device.type == "cuda"
and torch.cuda.current_device() != self.device.index)
or (self.device.type == "mps")):
suffixes.append("device='" + str(self.device) + "'")
# Tensor printing performs tensor operations like slice, indexing, etc to make it in a
# representable format. These operations on ipu/xla/lazy tensor results in compilations. Hence,
# to avoid compilations, copying the tensor to cpu before printing.
if self.device.type in ["xla", "lazy", "ipu"]:
self = self.to("cpu")
# TODO: add an API to map real -> complex dtypes
_default_complex_dtype = (torch.cdouble if torch.get_default_dtype()
== torch.double else torch.cfloat)
has_default_dtype = self.dtype in (
torch.get_default_dtype(),
_default_complex_dtype,
torch.int64,
torch.bool,
)
if self.is_sparse:
suffixes.append("size=" + str(tuple(self.shape)))
from torch._subclasses.fake_tensor import FakeTensor
if not self.is_meta and not isinstance(self, FakeTensor):
suffixes.append("nnz=" + str(self._nnz()))
if not has_default_dtype:
suffixes.append("dtype=" + str(self.dtype))
if not custom_contents_provided:
indices_prefix = "indices=tensor("
indices = self._indices().detach()
indices_str = _tensor_str(indices, indent + len(indices_prefix))
if indices.numel() == 0:
indices_str += ", size=" + str(tuple(indices.shape))
values_prefix = "values=tensor("
values = self._values().detach()
values_str = _tensor_str(values, indent + len(values_prefix))
if values.numel() == 0:
values_str += ", size=" + str(tuple(values.shape))
tensor_str = (indices_prefix + indices_str + "),\n" +
" " * indent + values_prefix + values_str + ")")
elif self.layout in {
torch.sparse_csr,
torch.sparse_csc,
torch.sparse_bsr,
torch.sparse_bsc,
}:
suffixes.append("size=" + str(tuple(self.shape)))
suffixes.append("nnz=" + str(self._nnz()))
if not has_default_dtype:
suffixes.append("dtype=" + str(self.dtype))
if not custom_contents_provided:
compressed_indices_method, plain_indices_method = {
torch.sparse_csr:
(torch.Tensor.crow_indices, torch.Tensor.col_indices),
torch.sparse_csc:
(torch.Tensor.ccol_indices, torch.Tensor.row_indices),
torch.sparse_bsr:
(torch.Tensor.crow_indices, torch.Tensor.col_indices),
torch.sparse_bsc:
(torch.Tensor.ccol_indices, torch.Tensor.row_indices),
}[self.layout]
if self.layout in {torch.sparse_csr, torch.sparse_bsr}:
cdimname, pdimname = "row", "column"
else:
cdimname, pdimname = "column", "row"
compressed_indices_prefix = f"c{cdimname[:3]}_indices=tensor("
compressed_indices = compressed_indices_method(self).detach()
compressed_indices_str = _tensor_str(
compressed_indices, indent + len(compressed_indices_prefix))
if compressed_indices.numel() == 0:
compressed_indices_str += ", size=" + str(
tuple(compressed_indices.shape))
plain_indices_prefix = f"{pdimname[:3]}_indices=tensor("
plain_indices = plain_indices_method(self).detach()
plain_indices_str = _tensor_str(plain_indices,
indent + len(plain_indices_prefix))
if plain_indices.numel() == 0:
plain_indices_str += ", size=" + str(tuple(
plain_indices.shape))
values_prefix = "values=tensor("
values = self.values().detach()
values_str = _tensor_str(values, indent + len(values_prefix))
if values.numel() == 0:
values_str += ", size=" + str(tuple(values.shape))
tensor_str = (compressed_indices_prefix + compressed_indices_str +
"),\n" + " " * indent + plain_indices_prefix +
plain_indices_str + "),\n" + " " * indent +
values_prefix + values_str + ")")
elif self.is_quantized:
suffixes.append("size=" + str(tuple(self.shape)))
if not has_default_dtype:
suffixes.append("dtype=" + str(self.dtype))
suffixes.append("quantization_scheme=" + str(self.qscheme()))
if (self.qscheme() == torch.per_tensor_affine
or self.qscheme() == torch.per_tensor_symmetric):
suffixes.append("scale=" + str(self.q_scale()))
suffixes.append("zero_point=" + str(self.q_zero_point()))
elif (self.qscheme() == torch.per_channel_affine
or self.qscheme() == torch.per_channel_symmetric
or self.qscheme() == torch.per_channel_affine_float_qparams):
suffixes.append("scale=" + str(self.q_per_channel_scales()))
suffixes.append("zero_point=" +
str(self.q_per_channel_zero_points()))
suffixes.append("axis=" + str(self.q_per_channel_axis()))
if not custom_contents_provided:
tensor_str = _tensor_str(self.dequantize(), indent)
elif self.is_nested:
if not custom_contents_provided:
def indented_str(s, indent):
return "\n".join(f" {line}" for line in s.split("\n"))
strs = ",\n".join(
indented_str(str(t), indent + 1)
for t in torch.ops.aten.unbind.int(self, 0))
tensor_str = f"[\n{strs}\n]"
elif torch._is_functional_tensor(self):
prefix = "_to_functional_tensor("
tensor_str = repr(torch._from_functional_tensor(self))
else:
if self.is_meta:
suffixes.append("size=" + str(tuple(self.shape)))
if self.dtype != torch.get_default_dtype():
suffixes.append("dtype=" + str(self.dtype))
# TODO: This implies that ellipses is valid syntax for allocating
# a meta tensor, which it could be, but it isn't right now
if not custom_contents_provided:
tensor_str = "..."
else:
if self.numel() == 0 and not self.is_sparse:
# Explicitly print the shape if it is not (0,), to match NumPy behavior
if self.dim() != 1:
suffixes.append("size=" + str(tuple(self.shape)))
# In an empty tensor, there are no elements to infer if the dtype
# should be int64, so it must be shown explicitly.
if self.dtype != torch.get_default_dtype():
suffixes.append("dtype=" + str(self.dtype))
if not custom_contents_provided:
tensor_str = "[]"
else:
if not has_default_dtype:
suffixes.append("dtype=" + str(self.dtype))
if not custom_contents_provided:
if self.layout != torch.strided:
tensor_str = _tensor_str(self.to_dense(), indent)
else:
tensor_str = _tensor_str(self, indent)
if self.layout != torch.strided:
suffixes.append("layout=" + str(self.layout))
# Use inp here to get the original grad_fn and not the one generated by the forward grad
# unpacking.
if inp.grad_fn is not None:
name = type(inp.grad_fn).__name__
if name == "CppFunction":
name = inp.grad_fn.name().rsplit("::", 1)[-1]
suffixes.append("grad_fn=<{}>".format(name))
elif inp.requires_grad:
suffixes.append("requires_grad=True")
if self.has_names():
suffixes.append("names={}".format(self.names))
if tangent is not None:
suffixes.append("tangent={}".format(tangent))
string_repr = _add_suffixes(prefix + tensor_str,
suffixes,
indent,
force_newline=self.is_sparse)
# Check if this instance is flagged as a parameter and change the repr accordingly.
# Unfortunately, this function has to be aware of this detail.
# NB: This is currently skipped for plain tensor parameters to maintain BC. In the future,
# this should be done for those as well to produce a valid repr.
if isinstance(self, torch.nn.Parameter) and not is_plain_tensor:
string_repr = f"Parameter({string_repr})"
return string_repr
import torch data = np.array([1, 2, 3]) t1 = torch.Tensor(data) t2 = torch.tensor(data) t3 = torch.as_tensor(data) t4 = torch.from_numpy(data) print(t1) print(t2) print(t3) print(t4) print(type(t1), t1.dtype) print(type(t2), t2.dtype) print(type(t3), t3.dtype) print(type(t4), t4.dtype) print(torch.get_default_dtype()) # t1 = torch.tensor([1,2,3],dtype=torch.float64) print(t1, t1.dtype) data[0] = 0 data[1] = 0 data[2] = 0 print(t1) print(t2) print(t3) print(t4) print(type(t1), t1.dtype) print(type(t2), t2.dtype) print(type(t3), t3.dtype) print(type(t4), t4.dtype)
def fit(self,
*datasets,
learn_bias=True,
batch_size=None,
max_epochs=1,
dtype=None,
underdetermined=False,
dry_run=False):
'''Trains a least squares model.
Users should not call this method directly, but instead call the
estimator as a function.
.. todo::
:class:`LeastSquares` does not yet support CUDA.
Arguments:
datasets (Dataset):
The datasets to fit. If more than one are given, they are
combined using `toys.zip`. The target is taken from the last
column.
Keyword Arguments:
learn_bias (bool):
If true (the default), learn an additive bias/intercept term.
If false, the intercept is always 0.
batch_size (int):
The number of samples in each batch. This should be greater
than the number of input features. The default is to read the
entire dataset in a single batch.
max_epochs (int):
The number of times to iterate over the dataset.
dtype (str or torch.dtype):
Cast the module to this data type. This can be a PyTorch dtype
object, a conventional name like 'float' and 'double', or an
explicit name like 'float32' and 'float64'. The default is
determined by `torch.get_default_dtype` and may be set with
`torch.set_default_dtype`.
underdetermined (bool):
The estimator issues a warning if the problem is
underdetermined, i.e. the batch size is less than the number
of features. Set this to true to ignore the warning.
dry_run (bool):
If true, break from loops early. Useful for debugging.
Returns:
model (TorchModel):
A linear model minimizing the mean squared error. The model
expects $n$ inputs where $n$ is the number of feature columns
in the training data.
'''
dataset = toys.zip(*datasets)
dataset = toys.flatten(dataset, supervised=True)
batch_size = batch_size or len(dataset)
batch_size = min(batch_size, len(dataset))
dtype = dtype or torch.get_default_dtype()
dtype = parse_dtype(dtype)
x, y = dataset[0]
in_features = len(x)
out_features = len(y)
if not underdetermined and batch_size < in_features:
logger.warning('least squares problem is underdetermined')
logger.warning(
f' this means that the batch size is less than the number of features'
)
logger.warning(
f' batch_size={batch_size}, features={in_features}')
logger.warning(
f' set underdetermined=True to disable this warning')
if learn_bias:
x_mean = Mean(dim=0)
y_mean = Mean(dim=0)
for x, y in toys.batches(dataset, batch_size):
x_mean.accumulate(x)
y_mean.accumulate(y)
x_mean = x_mean.reduce()
y_mean = y_mean.reduce()
weight = Mean(dim=0)
for i in range(max_epochs):
for x, y in toys.batches(dataset, batch_size):
if learn_bias:
x -= x_mean
y -= y_mean
# gels is the LAPACK routine for least squares.
# https://pytorch.org/docs/stable/torch.html#torch.gels
# https://software.intel.com/en-us/mkl-developer-reference-c-gels
batch_weight, _ = torch.gels(y, x)
batch_weight = batch_weight[:in_features]
assert batch_weight.shape == (in_features, out_features)
# We duplicate the solution for this batch to weight it by the
# batch size. This has no memory overhead, see `Tensor.expand`.
batch_weight = batch_weight.unsqueeze(0)
batch_weight = batch_weight.expand(len(x), in_features,
out_features)
weight.accumulate(batch_weight)
if dry_run:
break
weight = weight.reduce()
bias = y_mean - x_mean @ weight if learn_bias else 0
mod = LeastSquaresModule(weight, bias)
mod = TorchModel(mod, device_ids=[], dtype=dtype)
mod = mod.eval()
return mod
def test_numpy_array_binary_ufunc_promotion(self, device, dtypes):
import operator
np_type = torch_to_numpy_dtype_dict[dtypes[0]]
torch_type = dtypes[1]
t = torch.tensor((1, ), device=device, dtype=torch_type)
a = np.array((1, ), dtype=np_type)
a_as_t = torch.from_numpy(a).to(device=device)
for np_first in (True, False):
for op in (operator.add, torch.add):
# Acquires results of binary ufunc type promotion.
try:
actual = op(a, t) if np_first else op(t, a)
except Exception as e:
actual = e
try:
expected = op(a_as_t, t) if np_first else op(t, a_as_t)
except Exception as e:
expected = e
same_result = (type(expected)
== type(actual)) and expected == actual
# Note: An "undesired failure," as opposed to an "expected failure"
# is both expected (we know the test will fail) and
# undesirable (if PyTorch was working properly the test would
# not fail). This test is affected by three issues (see below)
# that will cause undesired failures. It detects when these
# issues will occur and updates this bool accordingly.
undesired_failure = False
# A NumPy array as the first argument to the plus operator
# or as any argument to torch.add is not working as
# intended.
# See https://github.com/pytorch/pytorch/issues/36363.
if np_first and op is operator.add:
undesired_failure = True
if op is torch.add:
undesired_failure = True
# Expects the same result if undesired_failure is false
# and a different result otherwise.
# Note: These cases prettyprint the failing inputs to make
# debugging test failures easier.
if undesired_failure and same_result:
msg = (
"Failure: {0} == {1}. "
"torch type was {2}. NumPy type was {3}. np_first is {4} "
"default type is {5}.").format(
actual, expected, torch_type, np_type, np_first,
torch.get_default_dtype())
self.fail(msg)
if not undesired_failure and not same_result:
msg = (
"Failure: {0} != {1}. "
"torch type was {2}. NumPy type was {3}. np_first is {4} "
"default type is {5}.").format(
actual, expected, torch_type, np_type, np_first,
torch.get_default_dtype())
self.fail(msg)
def array(self, arr, dtype=None):
""" create an array from an array-like sequence """
if dtype is None:
dtype = torch.get_default_dtype()
return torch.tensor(arr, device="cuda", dtype=dtype)
class TorchBackend(Backend):
""" Torch Backend """
# types
int = torch.int64
""" integer type for array"""
float = torch.get_default_dtype()
""" floating type for array """
# methods
exp = staticmethod(torch.exp)
""" exponential of all elements in array """
sin = staticmethod(torch.sin)
""" sine of all elements in array """
cos = staticmethod(torch.cos)
""" cosine of all elements in array """
sum = staticmethod(torch.sum)
""" sum elements in array """
stack = staticmethod(torch.stack)
""" stack multiple arrays """
@staticmethod
def transpose(arr, axes=None):
""" transpose array by flipping two dimensions """
if axes is None:
axes = tuple(range(len(arr.shape) - 1, -1, -1))
return arr.permute(*axes)
squeeze = staticmethod(torch.squeeze)
""" remove dim-1 dimensions """
reshape = staticmethod(torch.reshape)
""" reshape array into given shape """
bmm = staticmethod(torch.bmm)
""" batch matrix multiply two arrays """
@staticmethod
def is_array(arr):
""" check if an object is an array """
def array(self, arr, dtype=None):
""" create an array from an array-like sequence """
if dtype is None:
dtype = torch.get_default_dtype()
return torch.tensor(arr, device="cpu", dtype=dtype)
return torch.is_tensor(arr)
# constructors
ones = staticmethod(torch.ones)
""" create an array filled with ones """
zeros = staticmethod(torch.zeros)
""" create an array filled with zeros """
def linspace(self, start, stop, num=50, endpoint=True):
""" create a linearly spaced array between two points """
delta = (stop - start) / float(num - float(endpoint))
if not delta:
return self.array([start] * num)
return torch.arange(start, stop + 0.5 * float(endpoint) * delta,
delta)
arange = staticmethod(torch.arange)
""" create a range of values """
def numpy(self, arr):
""" convert the array to numpy array """
if torch.is_tensor(arr):
return arr.numpy()
else:
return numpy.asarray(arr)
def __init__(
self,
sampling_rate: int = 16000,
frame_length: Seconds = 0.025,
frame_shift: Seconds = 0.01,
fft_length: int = 512,
remove_dc_offset: bool = True,
preemph_coeff: float = 0.97,
window_type: str = "povey",
dither: float = 0.0,
snip_edges: bool = False,
energy_floor: float = EPSILON,
raw_energy: bool = True,
use_energy: bool = False,
use_fft_mag: bool = False,
low_freq: float = 20.0,
high_freq: float = -400.0,
num_filters: int = 23,
norm_filters: bool = False,
num_ceps: int = 13,
cepstral_lifter: int = 22,
):
super().__init__(
sampling_rate,
frame_length,
frame_shift,
fft_length,
remove_dc_offset=remove_dc_offset,
preemph_coeff=preemph_coeff,
window_type=window_type,
dither=dither,
snip_edges=snip_edges,
energy_floor=energy_floor,
raw_energy=raw_energy,
use_energy=use_energy,
)
self.use_fft_mag = use_fft_mag
self.low_freq = low_freq
self.high_freq = high_freq
self.num_filters = num_filters
self.norm_filters = norm_filters
self.num_ceps = num_ceps
self.cepstral_lifter = cepstral_lifter
if use_fft_mag:
self._to_spec = _spectrogram
else:
self._to_spec = _pow_spectrogram
fb = create_mel_scale(
num_filters=num_filters,
fft_length=fft_length,
sampling_rate=sampling_rate,
low_freq=low_freq,
high_freq=high_freq,
norm_filters=norm_filters,
)
self._fb = nn.Parameter(
torch.tensor(fb, dtype=torch.get_default_dtype()), requires_grad=False
)
self._dct = nn.Parameter(
self.make_dct_matrix(self.num_ceps, self.num_filters), requires_grad=False
)
self._lifter = nn.Parameter(
self.make_lifter(self.num_ceps, self.cepstral_lifter), requires_grad=False
)
def __init__(
self,
Rs_in1,
Rs_in2,
Rs_out,
instr: List[Tuple[int, int, int, str, bool]],
normalization: str = 'component',
own_weight: bool = True,
weight_batch: bool = False,
_specialized_code=True,
):
"""
Create a Tensor Product operation that has each of his path weighted by a parameter.
`instr` is a list of instructions.
An instruction if of the form (i_1, i_2, i_out, mode, weight)
it means "Put `Rs_in1[i_1] otimes Rs_in2[i_2] into Rs_out[i_out]"
`mode` determines the way the multiplicities are treated.
`weight` determines if weights has to be used.
The default mode should be 'uvw', meaning that all paths are created.
"""
super().__init__()
assert normalization in ['component', 'norm'], normalization
self.Rs_in1 = o3.convention(Rs_in1)
self.Rs_in2 = o3.convention(Rs_in2)
self.Rs_out = o3.convention(Rs_out)
code = f"""
import torch
@torch.jit.script
def main(< w3j >x1: torch.Tensor, x2: torch.Tensor, w: torch.Tensor) -> torch.Tensor:
batch = x1.shape[0]
out = x1.new_zeros((batch, {o3.dim(self.Rs_out)}))
ein = torch.einsum
"""
index_w = 0
wigners = set()
count = [0 for _ in range(o3.dim(self.Rs_out))]
instr = sorted(instr) # for optimization
for i_1, (mul_1, l_1, p_1) in enumerate(self.Rs_in1):
index_1 = o3.dim(self.Rs_in1[:i_1])
dim_1 = mul_1 * (2 * l_1 + 1)
code += f" x1_{i_1} = x1[:, {index_1}:{index_1+dim_1}].reshape(batch, {mul_1}, {2 * l_1 + 1})\n"
code += f"\n"
for i_2, (mul_2, l_2, p_2) in enumerate(self.Rs_in2):
index_2 = o3.dim(self.Rs_in2[:i_2])
dim_2 = mul_2 * (2 * l_2 + 1)
code += f" x2_{i_2} = x2[:, {index_2}:{index_2+dim_2}].reshape(batch, {mul_2}, {2 * l_2 + 1})\n"
code += f"\n"
last_ss = None
for i_1, i_2, i_out, mode, weight in instr:
mul_1, l_1, p_1 = self.Rs_in1[i_1]
mul_2, l_2, p_2 = self.Rs_in2[i_2]
mul_out, l_out, p_out = self.Rs_out[i_out]
dim_1 = mul_1 * (2 * l_1 + 1)
dim_2 = mul_2 * (2 * l_2 + 1)
dim_out = mul_out * (2 * l_out + 1)
index_1 = o3.dim(self.Rs_in1[:i_1])
index_2 = o3.dim(self.Rs_in2[:i_2])
index_out = o3.dim(self.Rs_out[:i_out])
assert p_1 * p_2 == p_out
assert abs(l_1 - l_2) <= l_out <= l_1 + l_2
if dim_1 == 0 or dim_2 == 0 or dim_out == 0:
continue
code += (
f" with torch.autograd.profiler.record_function("
f"'{o3.format_Rs([self.Rs_in1[i_1]])} x {o3.format_Rs([self.Rs_in2[i_2]])} "
f"= {o3.format_Rs([self.Rs_out[i_out]])} {mode} {weight}'):\n")
code += f" s1 = x1_{i_1}\n"
code += f" s2 = x2_{i_2}\n"
assert mode in ['uvw', 'uvu', 'uvv', 'uuw', 'uuu', 'uvuv']
if _specialized_code:
# optimized code for special cases:
# 0 x 0 = 0
# 0 x L = L
# L x 0 = L
# L x L = 0
# 1 x 1 = 1
if (l_1, l_2, l_out) == (0, 0, 0) and mode in [
'uvw', 'uvu'
] and normalization in ['component', 'norm'] and weight:
code += f" s1 = s1.reshape(batch, {mul_1})\n"
code += f" s2 = s2.reshape(batch, {mul_2})\n"
if mode == 'uvw':
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,zu,zv->zw', sw, s1, s2)\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
if mode == 'uvu':
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,zu,zv->zu', sw, s1, s2)\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
continue
if l_1 == 0 and l_2 == l_out and mode in [
'uvw', 'uvu'
] and normalization == 'component' and weight:
code += f" s1 = s1.reshape(batch, {mul_1})\n"
if mode == 'uvw':
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,zu,zvi->zwi', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
if mode == 'uvu':
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,zu,zvi->zui', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
continue
if l_2 == 0 and l_1 == l_out and mode in [
'uvw', 'uvu'
] and normalization == 'component' and weight:
code += f" s2 = s2.reshape(batch, {mul_2})\n"
if mode == 'uvw':
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,zui,zv->zwi', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
if mode == 'uvu':
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,zui,zv->zui', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
continue
if l_1 == l_2 and l_out == 0 and mode == 'uvw' and normalization == 'component' and weight:
# Cl_l_0 = eye(3) / sqrt(2L+1)
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out}).div({(2 * l_1 + 1)**0.5})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,zui,zvi->zw', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
continue
if l_1 == l_2 and l_out == 0 and mode == 'uvu' and normalization == 'component' and weight:
# Cl_l_0 = eye(3) / sqrt(2L+1)
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}).div({(2 * l_1 + 1)**0.5})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,zui,zvi->zu', sw, s1, s2).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
continue
if (l_1, l_2, l_out) == (
1, 1, 1
) and mode == 'uvw' and normalization == 'component' and weight:
# C1_1_1 = levi-civita / sqrt(2)
code += f" s1 = s1.reshape(batch, {mul_1}, 1, {2 * l_1 + 1})\n"
code += f" s2 = s2.reshape(batch, 1, {mul_2}, {2 * l_2 + 1})\n"
code += f" s1, s2 = torch.broadcast_tensors(s1, s2)\n"
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out}).div({2**0.5})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,zuvi->zwi', sw, torch.cross(s1, s2, dim=3)).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
continue
if (l_1, l_2, l_out) == (
1, 1, 1
) and mode == 'uvu' and normalization == 'component' and weight:
# C1_1_1 = levi-civita / sqrt(2)
code += f" s1 = s1.reshape(batch, {mul_1}, 1, {2 * l_1 + 1})\n"
code += f" s2 = s2.reshape(batch, 1, {mul_2}, {2 * l_2 + 1})\n"
code += f" s1, s2 = torch.broadcast_tensors(s1, s2)\n"
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}).div({2**0.5})\n"
index_w += dim_w
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,zuvi->zui', sw, torch.cross(s1, s2, dim=3)).reshape(batch, {dim_out})\n"
code += "\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
continue
if last_ss != (i_1, i_2, mode[:2]):
if mode[:2] == 'uv':
code += f" ss = ein('zui,zvj->zuvij', s1, s2)\n"
if mode[:2] == 'uu':
code += f" ss = ein('zui,zuj->zuij', s1, s2)\n"
last_ss = (i_1, i_2, mode[:2])
wigners.add((l_1, l_2, l_out))
if mode == 'uvw':
assert weight
dim_w = mul_1 * mul_2 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2}, {mul_out})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uvw,ijk,zuvij->zwk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1 * mul_2
if mode == 'uvu':
assert mul_1 == mul_out
if weight:
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,ijk,zuvij->zuk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
else:
dim_w = 0
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('ijk,zuvij->zuk', C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_2
if mode == 'uvv':
assert mul_2 == mul_out
if weight:
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,ijk,zuvij->zvk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
else:
dim_w = 0
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('ijk,zuvij->zvk', C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1
if mode == 'uuw':
assert mul_1 == mul_2
assert weight
dim_w = mul_1 * mul_out
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_out})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uw,ijk,zuij->zwk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += mul_1
if mode == 'uuu':
assert mul_1 == mul_2 == mul_out
if weight:
dim_w = mul_1
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >u,ijk,zuij->zuk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
else:
dim_w = 0
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('ijk,zuij->zuk', C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += 1
if mode == 'uvuv':
assert mul_1 * mul_2 == mul_out
if weight:
dim_w = mul_1 * mul_2
code += f" sw = w[< weight index >{index_w}:{index_w+dim_w}].reshape(< weight shape >{mul_1}, {mul_2})\n"
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('< weight sym >uv,ijk,zuvij->zuvk', sw, C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
else:
dim_w = 0
code += f" out[:, {index_out}:{index_out+dim_out}] += ein('ijk,zuvij->zuvk', C{l_1}_{l_2}_{l_out}, ss).reshape(batch, {dim_out})\n"
for pos in range(index_out, index_out + dim_out):
count[pos] += 1
index_w += dim_w
code += "\n"
ilast = 0
clast = count[0]
for i, c in enumerate(count):
if clast != c:
if clast > 1:
code += f" out[:, {ilast}:{i}].div_({clast ** 0.5})\n"
clast = c
ilast = i
if clast > 1:
code += f" out[:, {ilast}:].div_({clast ** 0.5})\n"
wigners = sorted(wigners)
self.wigners_names = [
f"C{l_1}_{l_2}_{l_3}" for l_1, l_2, l_3 in wigners
]
args = ", ".join(f"{arg}: torch.Tensor" for arg in self.wigners_names)
if args:
args += ', '
for arg, (l_1, l_2, l_out) in zip(self.wigners_names, wigners):
wig = o3.wigner_3j(l_1, l_2, l_out)
if normalization == 'component':
wig *= (2 * l_out + 1)**0.5
if normalization == 'norm':
wig *= (2 * l_1 + 1)**0.5 * (2 * l_2 + 1)**0.5
self.register_buffer(arg, wig)
code += f" return out"
code = code.replace("< w3j >", args)
if index_w == 0:
code = code.replace(', w: torch.Tensor', '')
self.weight_batch = weight_batch
if self.weight_batch:
code = code.replace('< weight index >', ':, ')
code = code.replace('< weight shape >', 'batch, ')
code = code.replace('< weight sym >', 'z')
else:
code = code.replace('< weight index >', '')
code = code.replace('< weight shape >', '')
code = code.replace('< weight sym >', '')
self.code = code
self.main = eval_code(self.code).main
self.nweight = index_w
if own_weight:
assert not self.weight_batch, "weight_batch and own_weight are incompatible"
self.weight = torch.nn.Parameter(torch.randn(self.nweight))
self.to(dtype=torch.get_default_dtype())
net = ResUNet(os.path.join(save_model_dir, pretrained_params)).cuda(device)
# net = ResUNetFPN(os.path.join(save_model_dir, pretrained_params)).cuda(device)
criterion = BCEDICELoss().cuda(device)
optimizer = optim.SGD(net.parameters(),
lr=lr,
momentum=0.9,
weight_decay=0.0005)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[10, 30, 50],
gamma=0.1)
print('%s Finish Preprocessing' % (datetime.now().ctime()))
print('%s Start Training' % (datetime.now().ctime()))
print(
'Dtype[%s] Base lr[%f] Epochs[%d] Dataset: train[%d] valid[%d] Iter: train[%d] valid[%d] Batch Size[%d]'
% (str(torch.get_default_dtype()), optimizer.param_groups[0]['lr'], epochs,
len(train_dataset), len(valid_dataset), len(train_dataloader),
len(valid_dataloader), batch_size))
valid(valid_dataloader, net, criterion, device)
for epoch in range(epochs):
scheduler.step()
train(train_dataloader, net, criterion, device, epoch, optimizer,
batch_size, print_interval)
valid_detail = valid(valid_dataloader, net, criterion, device)
save_model(net, save_model_dir, 'seg', epoch, valid_detail)
print('%s Finish Training' % (datetime.now().ctime()))
def elementwise_dtypes(
*_args,
type_promotion_kind: ELEMENTWISE_TYPE_PROMOTION_KIND,
) -> Tuple[torch.dtype, torch.dtype]:
"""
Computes the computation and result dtypes for elementwise type promotion
on the given arguments and with the given elementwise type promotion kind.
Note that not all inputs to an elementwise operation necessarily participate in type promotion.
For example, the "alpha" parameter of torch.add does not participate in type promotion,
although it may be cast to the Python type corresponding to the computation dtype that
the type promotion algorithm determines.
Default elementwise type promotion, which all other type promotion kinds tweak (see below),
first decides which of four ordered types to use:
bool -> integer -> floating point -> complex
The selected type is the "lowest" type in the above list such that all number arguments
have a weakly "lower" type and all tensor arguments have a weakly lower corresponding
type for their dtype.
Once the type is determined, the particular result dtype is found. The dtypes are
partially ordered as follows:
bool -> uint8, int8 -> int16 -> int32 -> int64 ->
float16, bfloat16 -> float32 -> float64 -> complex32 -> complex64 -> complex128
The result dtype is selected by:
- if no tensor's dtype has the same corresponding type as the one selected,
then the result dtype is the (default) dtype corresponding to the selected type
(for example, 1.5 + an integer tensor has a result dtype of the default floating point dtype)
- if the result type is complex then the dtype is:
- the default complex dtype if there are no floating point or complex tensors
- if there are floating point or complex tensors with one or more dimensions, then
the complex dtype corresponding to the highest corresponding complex dtype among those tensors
(for example, double + cfloat -> cdouble)
- if there are only floating point or complex tensors with zero dimensions, then
the complex dtype corresponding to the highest corresponding complex dtype among those tensors
- if the first two cases do not apply, the result dtype is the highest dtype among
all tensors with one or more dimensions of the output type, and if there are no such
tensors then it's the highest dtype among all tensors with zero dimensions of the output type
(for example, long + half -> half, even if the half tensor has zero dimensions)
The "corresponding complex dtypes" are:
float16 -> complex32
bfloat16 -> complex64
float32 -> complex64
float64 -> complex128
complex32 -> complex32
complex64 -> complex64
complex128 -> complex128
The DEFAULT type promotion kind computes per above, and then uses the result dtype to pick a computation
dtype by mapping low precision floating point and complex dtypes as follows:
float16 -> float32
bfloat16 -> float32
complex32 -> complex64
This is referred to as "op math", and the NO_OPMATH type promotion kind disables this mapping, making the
computation dtype the same as the result dtype when it's selected. NO_OPMATH is appropriate for kernels
which perform no mathematical operations on their tensors (see below for examples).
The INT_TO_FLOAT type promotion kind maps boolean and integer maps result dtypes to the default floating point dtype,
and computation dtypes to the appropriate op math dtype.
The COMPLEX_TO_FLOAT type promotion kind maps complex result dtypes to the corresponding float dtype, following this
mapping:
complex32 -> float16
complex64 -> float32
complex128 -> float64
Note that COMPLEX_TO_FLOAT derives the computation dtype as the DEFAULT setting does.
The BOOL_TO_LONG type promotion kind maps boolean computation and result dtypes to long.
The ALWAYS_BOOL type promotion kind always sets the result dtype to bool.
Example operators for each type promotion option:
DEFAULT : add
NO_OPMATH : where, nextafter, cat
INT_TO_FLOAT : sin
COMPLEX_TO_FLOAT : abs
BOOL_TO_LONG : pow
ALWAYS_BOOL : eq
"""
args = tuple(x for x in _args if x is not None)
highest_type: type = bool
for x in args:
if not isinstance(x, (Number, TensorLike)):
msg = (
"Unexpected type {0} when computing elementwise type promotion!"
.format(str(type(x))))
raise ValueError(msg)
if isinstance(x, Number):
highest_type = get_higher_type(highest_type, type(x))
else:
# x is a TensorLike
highest_type = get_higher_type(highest_type,
dtype_to_type(x.dtype))
result_dtype = None
def _find_highest_dtype_filtered(args,
filter,
*,
float_as_complex=False
) -> Optional[torch.dtype]:
zero_dim_tensor_dtype = None
one_plus_dim_tensor_dtype = None
for x in args:
if isinstance(x, TensorLike) and filter(x.dtype):
_dtype = x.dtype
if float_as_complex and is_float_dtype(_dtype):
_dtype = corresponding_complex_dtype(_dtype)
if x.ndim == 0:
zero_dim_tensor_dtype = get_higher_dtype(
zero_dim_tensor_dtype, _dtype)
else:
# x.ndim > 0
one_plus_dim_tensor_dtype = get_higher_dtype(
one_plus_dim_tensor_dtype, _dtype)
# Prefers dtype of tensors with one or more dimensions
if one_plus_dim_tensor_dtype is not None:
return one_plus_dim_tensor_dtype
return zero_dim_tensor_dtype
if highest_type is float:
result_dtype = _find_highest_dtype_filtered(args, is_float_dtype)
result_dtype = (torch.get_default_dtype()
if result_dtype is None else result_dtype)
elif highest_type is complex:
result_dtype = _find_highest_dtype_filtered(
args,
lambda x: is_float_dtype(x) or is_complex_dtype(x),
float_as_complex=True,
)
if result_dtype is None:
result_dtype = corresponding_complex_dtype(
torch.get_default_dtype())
elif highest_type is int:
result_dtype = _find_highest_dtype_filtered(args, is_integer_dtype)
result_dtype = torch.long if result_dtype is None else result_dtype
else:
# highest_type is bool
result_dtype = torch.bool
if type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.DEFAULT:
return get_computation_dtype(result_dtype), result_dtype
elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.NO_OPMATH:
return result_dtype, result_dtype
elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.INT_TO_FLOAT:
if is_integer_dtype(result_dtype) or is_boolean_dtype(result_dtype):
result_dtype = torch.get_default_dtype()
return get_computation_dtype(result_dtype), result_dtype
elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT:
# NOTE: computation can still occur in a complex dtype
computation_dtype = get_computation_dtype(result_dtype)
if is_complex_dtype(result_dtype):
result_dtype = corresponding_real_dtype(result_dtype)
return computation_dtype, result_dtype
elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.BOOL_TO_LONG:
if is_boolean_dtype(result_dtype):
return torch.long, torch.long
return get_computation_dtype(result_dtype), result_dtype
elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.ALWAYS_BOOL:
return get_computation_dtype(result_dtype), torch.bool
else:
raise ValueError("Unknown type promotion kind {0}".format(
str(type_promotion_kind)))
def main(args):
print("Preparation...")
m5 = M5Data()
# get non-aggregated sales of all items from all Walmart stores
data = m5.get_aggregated_sales(m5.aggregation_levels[-1])
# reshape into num_stores x num_products x duration x 1
data = data.reshape(10, 3049, -1, 1)
T0 = 37 + 28 * 3 # begining, skip a small period to calculate moving average
T2 = data.size(-2) + 28 # end + submission-interval
T1 = T2 - 28 # train/test split
assert (T2 - T0) % 28 == 0
covariates = torch.arange(T2).unsqueeze(-1)
# extra covariates (see explanations in Model constructor)
snap = m5.get_snap().repeat_interleave(torch.tensor([4, 3, 3]), dim=-1)
snap = snap.t().unsqueeze(1).unsqueeze(-1)
dept = m5.get_dummy_dept().reshape(10, -1, 7).unsqueeze(-2)
saled = (m5.get_prices() != 0).type(torch.get_default_dtype()).reshape(
10, 3049, -1, 1)
saled = saled * (1 - m5.get_christmas())
ma28x1 = data.unfold(-2, 28 * 1, 1).mean(-1)
ma28x1 = torch.nn.functional.pad(ma28x1, (0, 0, 27 + 28 * 1, 0))
ma28x2 = data.unfold(-2, 28 * 2, 1).mean(-1)
ma28x2 = torch.nn.functional.pad(ma28x2, (0, 0, 27 + 28 * 2, 0))
ma28x3 = data.unfold(-2, 28 * 3, 1).mean(-1)
ma28x3 = torch.nn.functional.pad(ma28x3, (0, 0, 27 + 28 * 3, 0))
log_ma = torch.cat([ma28x1, ma28x2, ma28x3], -1).clamp(min=1e-3).log()
del ma28x1, ma28x2, ma28x3 # save memory
data = data.clamp(min=1e-3).to(args.device)
covariates = covariates.to(args.device)
snap = snap.to(args.device)
dept = dept.to(args.device)
saled = saled.to(args.device)
log_ma = log_ma.to(args.device)
if data.is_cuda:
torch.set_default_tensor_type(torch.cuda.FloatTensor)
def transform(pred, truth):
# our pred/truth are timeseries at non-aggregated level;
# we will aggregate them to all aggregation levels before evaluation.
num_samples, duration = pred.size(0), pred.size(-2)
pred = pred.reshape(num_samples, -1, duration)
truth = truth.round().reshape(-1, duration).cpu()
agg_pred = m5.aggregate_samples(pred, *m5.aggregation_levels)
agg_truth = m5.aggregate_samples(truth.unsqueeze(0),
*m5.aggregation_levels).squeeze(0)
return agg_pred.unsqueeze(-1), agg_truth.unsqueeze(-1)
def forecaster_options_fn(t0=None, t1=None, t2=None):
forecaster_options = {
"create_plates": create_plates,
"learning_rate": args.learning_rate,
"learning_rate_decay": args.learning_rate_decay,
"clip_norm": args.clip_norm,
"num_steps": args.num_steps,
"log_every": args.log_every,
"guide": NormalGuide(create_plates),
}
return forecaster_options
if args.submit:
pyro.set_rng_seed(args.seed)
print("Training...")
forecaster = M5Forecaster(Model(snap, dept, saled, log_ma),
data[:, :, T0:T1], covariates[T0:T1],
**forecaster_options_fn())
print("Forecasting...")
samples = forecaster(data[:, :, T0:T1],
covariates[T0:T2],
num_samples=1000,
batch_size=10)
samples = samples.reshape(-1, m5.num_timeseries, 28)
agg_samples = m5.aggregate_samples(samples, *m5.aggregation_levels)
# cast to numpy because pyro quantile implementation is memory hungry
print("Calculate quantiles...")
q = np.quantile(agg_samples.numpy(), m5.quantiles, axis=0)
print("Make submission...")
m5.make_uncertainty_submission(args.output_file,
q,
float_format='%.3f')
else:
# In this branch, we do backtesting.
# calculate weight of each timeseries
weight = m5.get_aggregated_ma_dollar_sales(
m5.aggregation_levels[-1]).cpu()
weight = weight / weight.sum(0, keepdim=True)
agg_weight = m5.aggregate_samples(weight.unsqueeze(0),
*m5.aggregation_levels).squeeze(0)
min_train_window = T1 - T0 - args.test_window - (args.num_windows -
1) * args.stride
print("Backtesting with skip window {}...".format(T0))
# we will skip crps because it is slow
metrics = {"mae": eval_mae, "rmse": eval_rmse, "pl": eval_pl}
windows = m5_backtest(data,
covariates[:T1],
lambda: Model(snap, dept, saled, log_ma),
weight=agg_weight,
skip_window=T0,
metrics=metrics,
transform=transform,
forecaster_fn=M5Forecaster,
min_train_window=min_train_window,
test_window=args.test_window,
stride=args.stride,
forecaster_options=forecaster_options_fn,
num_samples=1000,
batch_size=10,
seed=args.seed)
with open(args.output_file, "wb") as f:
pickle.dump(windows, f)
for metric in metrics:
ws_name = "ws_{}".format(metric)
values = torch.tensor([w[ws_name] for w in windows])
print("{} = {:0.3g} +- {:0.2g}".format(ws_name, values.mean(),
values.std()))
def __init__(self,
channel=1,
vel_amplitude=3,
vel_fwhm=20,
translation=20,
rotation=15,
zoom=0.15,
shear=0.012,
gmm_fwhm=10,
bias_amplitude=0.5,
bias_fwhm=50,
gamma=0.6,
motion_fwhm=3,
resolution=8,
noise=48,
gfactor_amplitude=0.5,
gfactor_fwhm=64,
gmm_cat='labels',
bag=0.5,
droppable_labels=None,
predicted_labels=None,
patch_size=None,
dtype=None):
"""
Parameters
----------
channel : int, default=1
Number of channels
vel_amplitude : float, default=3
Amplitude of the velocity field (larger -> larger displacement)
vel_fwhm : float, default=20
Full-width at half-maximum of the velocity field
(larger -> smoother deformation field)
translation : float, default=20
Maximum translation (in highres voxels)
rotation : float, default=15
Maximum rotation (in degrees)
zoom : float, default=0.15
Maximum scaling below/above 1
shear : float, default=0.012
Maximum shear factor
gmm_fwhm : float, default=10
Full-width at half-maximum of the within-tissue smoothing kernel
bias_amplitude : float, default=0.5
Amplitude of the bias field (larger -> stronger bias)
bias_fwhm : float, default=50
Full-width at half-maximum of the bias field
(larger -> smoother bias field)
gamma : float, default=0.6
Standard deviation of the log-normal distribution from
which the gamma exponent is sampled.
motion_fwhm : float, default=3
Maximum value of the full-width at half-maximum of the
smoothing filter simulating slice profile + motion
resolution : float, default=8
Maximum voxel size in the thick-slice direction
noise : float, default=48
Maximum value of the noise standard deviation
(assuming intensities in [0, 255])
gfactor_amplitude, default=0.5
Amplitude of the gfactor field (larger -> stronger bias)
The gfactor maps modulates the noise std to simulate
noise with non stationary variance (e.g. in parallel imaging).
gfactor_fwhm, default=64
Full-width at half-maximum of the gfactor field
(larger -> smoother bias field)
gmm_cat : {'labels', 'prob'}, default='labels'
Use hard labels or soft probability to sample the GMM.
Labels use less memory.
bag : float, default=0.5
Probability of sampling a bag/head class.
The bag class is generated by randomly dilating a brain mask.
droppable_labels : list[list[int]], optional
Groups of labels that can be randomly dropped before
synthesizing the image.
predicted_labels : list[int], optional
Labels that should be predicted.
There can be more labels used for synthesis than predicted.
patch_size : [list of] int, optional
Extract a random patch from the image
dtype : torch.dtype
"""
super().__init__()
self.gmm_cat = gmm_cat[0].lower()
dtype = dtype or torch.get_default_dtype()
self.droppable_labels = droppable_labels
self.predicted_labels = predicted_labels
# label 2 one hot
self.pbag = bag
self.bag = AddBagClass() if bag else (lambda x: x)
self.to_onehot = LabelToOneHot(dtype=dtype)
# deform
self.deform = RandomDeform(
amplitude='uniform',
amplitude_exp=vel_amplitude / 2,
amplitude_scale=vel_amplitude / 3.4,
fwhm='dirac',
fwhm_exp=vel_fwhm,
translation='uniform',
translation_scale=translation * 2 / 3.4,
rotation='uniform',
rotation_scale=rotation * 2 / 3.4,
zoom='uniform',
zoom_scale=zoom * 2 / 3.4,
shear='uniform',
shear_scale=shear * 2 / 3.4,
image_bound='nearest',
)
self.patch = RandomPatch(patch_size) if patch_size else None
# one hot 2 label
self.to_label = OneHotToLabel()
# gmm
self.mixture = HyperRandomGaussianMixture(
nb_classes=None,
nb_channels=channel,
means='uniform',
means_exp=125,
means_scale=250 / 3.46,
scales='uniform',
scales_exp=13,
scales_scale=24 / 3.46,
fwhm='uniform',
fwhm_exp=gmm_fwhm / 2,
fwhm_scale=gmm_fwhm / 3.46,
background_zero=True,
dtype=dtype,
)
# gamma
self.gamma = RandomGammaCorrection(
factor='lognormal',
factor_exp=1,
factor_scale=gamma,
vmin=0,
vmax=255,
)
# bias field
self.bias = HyperRandomBiasFieldTransform(
amplitude='uniform',
amplitude_exp=bias_amplitude / 2,
amplitude_scale=bias_amplitude / 3.4,
fwhm='dirac',
fwhm_exp=bias_fwhm,
)
# smooth (iso)
self.smooth = RandomSmooth(iso=True,
fwhm='uniform',
fwhm_exp=motion_fwhm / 2,
fwhm_scale=motion_fwhm / 3.46)
# smooth and downsample
self.lowres2d = RandomLowRes2D(resolution='uniform',
resolution_exp=resolution / 2 + 1,
resolution_scale=resolution / 3.46)
self.lowres3d = RandomLowRes3D(
resolution='uniform',
resolution_exp=(resolution**0.33) / 2 + 1,
resolution_scale=(resolution**0.33) / 3.46)
# add noise
gfactor = HyperRandomMultiplicativeField(
amplitude='uniform',
amplitude_exp=gfactor_amplitude / 2,
amplitude_scale=gfactor_amplitude / 3.46,
fwhm='dirac',
fwhm_exp=gfactor_fwhm,
dtype=dtype,
)
snoise = HyperRandomChiNoise(
sigma='uniform',
sigma_exp=noise / 2,
sigma_scale=noise / 3.46,
)
self.noise = lambda x: snoise(x, gfactor(x.shape, device=x.device))
# rescale
self.rescale = AffineQuantiles()
from ceem.opt_criteria import *
from ceem.ceem import CEEM
from ceem import logger
from src.datautils import get_us_county, smooth_and_diff
from src.models import ReactiveTime
import os
import click
opj = os.path.join
import logging
import seaborn as sns
torch.set_default_dtype(torch.float64)
dtype=torch.get_default_dtype()
@click.command()
@click.option('--show', '-s', type=int, default=0)
def main(show):
"""
Args:
show (int): will show the figure if show > 0
Notes:
Figure saved to figs/reactionfun.pdf
"""
torch.manual_seed(1)
# instantiate system
def estimate_knots_gaussian(data,
M,
above_noise,
weight=None,
edge_bins=0,
derivclip=None,
extrapolate='regression',
alpha=(0.9, 0.99),
KDE=True,
b_factor=1,
batchsize=None):
if not KDE and weight is not None:
raise NotImplementedError
start = 1. / (M - 2 * edge_bins + 1)
end = 1. - start
q1 = torch.linspace(start, end, M - 2 * edge_bins, device=data.device)
if edge_bins > 0:
start = start / (edge_bins + 1)
end = q1[0] - start
q0 = torch.linspace(start, end, edge_bins, device=data.device)
end = 1. - start
start = q1[-1] + start
q2 = torch.linspace(start, end, edge_bins, device=data.device)
q = torch.cat((q0, q1, q2), dim=0)
else:
q = q1
x = torch.randn(data.shape[1], M, device=data.device)
x = torch.sort(x, dim=1)[0]
y = x.clone()
deriv = torch.ones_like(x)
eps = 1e-5
for i in range(data.shape[1]):
if above_noise[i]:
if KDE:
rho = kde(data[:, i],
b_factor=b_factor,
weights=weight,
batchsize=batchsize)
scale = (rho.covariance[0, 0] + 1)**0.5
if weight is not None:
x[i] = quantile_weights(data[:, i], q, weight, scale)
else:
x[i] = quantile_weights(
data[:, i], q, torch.ones(len(data),
device=data.device), scale)
y[i] = 2**0.5 * scale * torch.erfinv(
2 * rho.cdf(x[i]).double() - 1).to(
torch.get_default_dtype())
dy = y[i, 1:] - y[i, :-1]
dx = x[i, 1:] - x[i, :-1]
while (dy <= eps).any() or (dx <= eps).any() or torch.isnan(
dy).any() or not torch.isfinite(y[i]).all():
select = torch.zeros(len(y[i]),
dtype=bool,
device=y.device)
select[1:] = dy <= eps
select[1:] += dx <= eps
select[1:] += torch.isnan(dy)
select += ~torch.isfinite(y[i])
x[i, select] = torch.rand(
torch.sum(select).item(), device=x.device) * (
torch.max(data[:, i]) - torch.min(data[:, i]) +
4 * scale) + torch.min(data[:, i]) - 2 * scale
x[i] = torch.sort(x[i])[0]
y[i] = 2**0.5 * scale * torch.erfinv(
2 * rho.cdf(x[i]).double() - 1).to(
torch.get_default_dtype())
dy = y[i, 1:] - y[i, :-1]
dx = x[i, 1:] - x[i, :-1]
else:
scale = eps
if weight is not None:
x[i] = quantile_weights(data[:, i], q, weight, scale)
else:
x[i] = quantile_weights(
data[:, i], q, torch.ones(len(data),
device=data.device), scale)
y[i] = 2**0.5 * torch.erfinv(2 * q.double() - 1).to(
torch.get_default_dtype())
dy = y[i, 1:] - y[i, :-1]
dx = x[i, 1:] - x[i, :-1]
q0 = q.clone()
while (dy <= eps).any() or (dx <= eps).any() or torch.isnan(
dy).any() or not torch.isfinite(y[i]).all():
select = torch.zeros(len(y[i]),
dtype=bool,
device=y.device)
select[1:] = dy <= eps
select[1:] += dx <= eps
select[1:] += torch.isnan(dy)
select += ~torch.isfinite(y[i])
q0[select] = torch.rand(torch.sum(select).item(),
device=q.device)
q0 = torch.sort(q0)[0]
x[i] = quantile_weights(
data[:, i], q0,
torch.ones(len(data), device=data.device), scale)
y[i] = 2**0.5 * torch.erfinv(2 * q0.double() - 1).to(
torch.get_default_dtype())
dy = y[i, 1:] - y[i, :-1]
dx = x[i, 1:] - x[i, :-1]
h = dx
s = dy / dx
deriv[i,
1:-1] = (s[:-1] * h[1:] + s[1:] * h[:-1]) / (h[1:] + h[:-1])
if derivclip == 1:
deriv[i, 0] = 1
deriv[i, -1] = 1
else:
if extrapolate == 'endpoint':
endx1 = torch.min(data[:, i])
endx2 = torch.max(data[:, i])
if KDE:
deriv[i, 0] = (
2**0.5 * scale *
torch.erfinv(2 * rho.cdf(endx1).double() - 1).to(
torch.get_default_dtype()) -
y[i, 0]) / (endx1 - x[i, 0])
deriv[i, -1] = (
2**0.5 * scale *
torch.erfinv(2 * rho.cdf(endx2).double() - 1).to(
torch.get_default_dtype()) -
y[i, -1]) / (endx2 - x[i, -1])
else:
deriv[i, 0] = (2**0.5 * torch.erfinv(
2 * torch.tensor(0.5 / len(data),
device=data.device,
dtype=torch.float64) - 1).to(
torch.get_default_dtype()) -
y[i, 0]) / (endx1 - x[i, 0])
deriv[i, -1] = (2**0.5 * torch.erfinv(
2 * torch.tensor(1 - 0.5 / len(data),
device=data.device,
dtype=torch.float64) - 1).to(
torch.get_default_dtype()) -
y[i, -1]) / (endx2 - x[i, -1])
elif extrapolate == 'regression':
endx1 = torch.sort(data[data[:, i] < x[i, 0], i])[0]
endx2 = torch.sort(data[data[:, i] > x[i, -1], i],
descending=True)[0]
if KDE:
if len(endx1) > 10:
endx1 = torch.quantile(
endx1,
torch.linspace(0, 1, 11,
device=endx1.device)[1:-1])
endy1 = 2**0.5 * scale * torch.erfinv(
2 * rho.cdf(endx1).double() - 1).to(
torch.get_default_dtype()) - y[i, 0]
if len(endx2) > 10:
endx2 = torch.quantile(
endx2,
torch.linspace(0, 1, 11,
device=endx1.device)[1:-1])
endy2 = 2**0.5 * scale * torch.erfinv(
2 * rho.cdf(endx2).double() - 1).to(
torch.get_default_dtype()) - y[i, -1]
else:
endy1 = 2**0.5 * torch.erfinv(2 * torch.linspace(
0.5,
len(endx1) - 0.5,
len(endx1),
device=data.device,
dtype=torch.float64) / len(data) - 1).to(
torch.get_default_dtype()) - y[i, 0]
endy2 = 2**0.5 * torch.erfinv(2 * (1 - torch.linspace(
0.5,
len(endx2) - 0.5,
len(endx2),
device=data.device,
dtype=torch.float64) / len(data)) - 1).to(
torch.get_default_dtype()) - y[i, -1]
endx1 -= x[i, 0]
select1 = torch.isfinite(endy1) & (endy1 < 0) & (endx1 < 0)
deriv[i, 0] = torch.sum(
endx1[select1] * endy1[select1]) / torch.sum(
endx1[select1] * endx1[select1])
endx2 -= x[i, -1]
select2 = torch.isfinite(endy2) & (endy2 > 0) & (endx2 > 0)
deriv[i, -1] = torch.sum(
endx2[select2] * endy2[select2]) / torch.sum(
endx2[select2] * endx2[select2])
if torch.sum(select1) == 0:
deriv[i, 0] = 1
if torch.sum(select2) == 0:
deriv[i, -1] = 1
y[i] = (1 - alpha[0]) * y[i] + alpha[0] * x[i]
deriv[i, 1:-1] = (1 - alpha[0]) * deriv[i, 1:-1] + alpha[0]
deriv[i, 0] = (1 - alpha[1]) * deriv[i, 0] + alpha[1]
deriv[i, -1] = (1 - alpha[1]) * deriv[i, -1] + alpha[1]
if derivclip is not None and derivclip > 1:
deriv[i, 0] = torch.clamp(deriv[i, 0], 1 / derivclip,
derivclip)
deriv[i, -1] = torch.clamp(deriv[i, -1], 1 / derivclip,
derivclip)
else:
dx = x[i, 1:] - x[i, :-1]
while (dx <= eps).any():
select = torch.zeros(len(x[i]), dtype=bool, device=x.device)
select[1:] = dx <= eps
x[i, select] = torch.rand(
torch.sum(select).item(), device=x.device) * (torch.max(
data[:, i]) - torch.min(data[:, i])) + torch.min(
data[:, i])
x[i] = torch.sort(x[i])[0]
y[i] = x[i]
dx = x[i, 1:] - x[i, :-1]
return x, y, deriv
def atom_dgl_multigraph(
atoms=None,
cutoff=8.0,
max_neighbors=12,
atom_features="cgcnn",
enforce_undirected=False,
max_attempts=3,
id=None,
):
"""Obtain a DGLGraph for Atoms object."""
all_neighbors = atoms.get_all_neighbors(r=cutoff)
# if a site has too few neighbors, increase the cutoff radius
min_nbrs = min(len(neighborlist) for neighborlist in all_neighbors)
# print('min_nbrs,max_neighbors=',min_nbrs,max_neighbors)
attempt = 0
while min_nbrs < max_neighbors:
print("extending cutoff radius!", attempt, cutoff, id)
lat = atoms.lattice
r_cut = max(cutoff, lat.a, lat.b, lat.c)
attempt += 1
if attempt >= max_attempts:
atoms = atoms.make_supercell([2, 2, 2])
print(
"Making supercell, exceeded,attempts",
max_attempts,
"cutoff",
r_cut,
id,
)
cutoff = r_cut
all_neighbors = atoms.get_all_neighbors(r=cutoff)
min_nbrs = min(len(neighborlist) for neighborlist in all_neighbors)
# return Graph.atom_dgl_multigraph(
# atoms, r_cut, max_neighbors, atom_features
# )
# build up edge list
# Currently there's no guarantee that this creates undirected graphs
# An undirected solution would build the full edge list where nodes are
# keyed by (index,image), and ensure each edge has a complementary edge
# indeed,JVASP-59628 is an example of a calculation where this produces
# a graph where one site has no incident edges!
# build an edge dictionary u -> v
# so later we can run through the dictionary
# and remove all pairs of edges
# so what's left is the odd ones out
edges = defaultdict(list)
u, v, r = [], [], []
for site_idx, neighborlist in enumerate(all_neighbors):
# sort on distance
neighborlist = sorted(neighborlist, key=lambda x: x[2])
ids = np.array([nbr[1] for nbr in neighborlist])
distances = np.array([nbr[2] for nbr in neighborlist])
c = np.array([nbr[3] for nbr in neighborlist])
# find the distance to the k-th nearest neighbor
max_dist = distances[max_neighbors - 1]
# keep all edges out to the neighbor shell of the k-th neighbor
ids = ids[distances <= max_dist]
c = c[distances <= max_dist]
distances = distances[distances <= max_dist]
u.append([site_idx] * len(ids))
v.append(ids)
r.append(distances)
# keep track of cell-resolved edges
# to enforce undirected graph construction
for dst, cell_id in zip(ids, c):
u_key = f"{site_idx}-(0.0, 0.0, 0.0)"
v_key = f"{dst}-{tuple(cell_id)}"
edge_key = tuple(sorted((u_key, v_key)))
edges[edge_key].append((site_idx, dst))
if enforce_undirected:
# add complementary edges to unpaired edges
for edge_pair in edges.values():
if len(edge_pair) == 1:
src, dst = edge_pair[0]
u.append(dst) # swap the order!
v.append(src)
r.append(atoms.raw_distance_matrix[src, dst])
u = torch.tensor(np.hstack(u))
v = torch.tensor(np.hstack(v))
r = torch.tensor(np.hstack(r)).type(torch.get_default_dtype())
# build up atom attribute tensor
species = atoms.elements
node_features = torch.tensor([
get_node_attributes(s, atom_features=atom_features)
for s in species
]).type(torch.get_default_dtype())
g = dgl.graph((u, v))
g.ndata["atom_features"] = node_features
g.edata["bondlength"] = r
return g
from typeguard import check_argument_types
from espnet2.enh.loss.criterions.tf_domain import FrequencyDomainMSE
from espnet2.enh.loss.criterions.time_domain import SISNRLoss
from espnet2.enh.loss.wrappers.pit_solver import PITSolver
from espnet2.fileio.sound_scp import SoundScpWriter
from espnet2.tasks.enh import EnhancementTask
from espnet2.tasks.enh_s2t import EnhS2TTask
from espnet2.torch_utils.device_funcs import to_device
from espnet2.torch_utils.set_all_random_seed import set_all_random_seed
from espnet2.train.abs_espnet_model import AbsESPnetModel
from espnet2.utils import config_argparse
from espnet2.utils.types import str2bool, str2triple_str, str_or_none
from espnet.utils.cli_utils import get_commandline_args
EPS = torch.finfo(torch.get_default_dtype()).eps
def get_train_config(train_config, model_file=None):
if train_config is None:
assert model_file is not None, (
"The argument 'model_file' must be provided "
"if the argument 'train_config' is not specified."
)
train_config = Path(model_file).parent / "config.yaml"
else:
train_config = Path(train_config)
return train_config
def recursive_dict_update(dict_org, dict_patch, verbose=False, log_prefix=""):
def __init__(self, *sizes, dtype=None, device=None):
super(ZeroLazyTensor, self).__init__(*sizes)
self.sizes = list(sizes)
self._dtype = dtype or torch.get_default_dtype()
self._device = device or torch.device("cpu")
def __init__(self,
use_cuda: bool,
noise_dim: [int, tuple],
halfspan_init: [float, int, to.Tensor],
halfspan_min: [float, to.Tensor] = 0.01,
train_mean: bool = False,
learnable: bool = True):
"""
Constructor
:param use_cuda: `True` to move the module to the GPU, `False` (default) to use the CPU
:param noise_dim: number of dimension
:param halfspan_init: initial value of the half interval for the exploration noise
:param halfspan_min: minimal value of the half interval for the exploration noise
:param train_mean: `True` if the noise should have an adaptive nonzero mean, `False` otherwise
:param learnable: `True` if the parameters should be tuneable (default), `False` for shallow use (just sampling)
"""
if not isinstance(halfspan_init, (float, int, to.Tensor)):
raise pyrado.TypeErr(given=halfspan_init,
expected_type=[float, to.Tensor])
if not (isinstance(halfspan_init, (int, float)) and halfspan_init > 0
or isinstance(halfspan_init, to.Tensor)
and all(halfspan_init > 0)):
raise pyrado.ValueErr(given=halfspan_init, g_constraint='0')
if not isinstance(halfspan_min, (float, to.Tensor)):
raise pyrado.TypeErr(given=halfspan_min,
expected_type=[float, to.Tensor])
if not (isinstance(halfspan_min, float) and halfspan_min > 0 or
isinstance(halfspan_min, to.Tensor) and all(halfspan_min > 0)):
raise pyrado.ValueErr(given=halfspan_min, g_constraint='0')
# Call torch.nn.Module's constructor
super().__init__()
if not use_cuda:
self._device = 'cpu'
elif use_cuda and to.cuda.is_available():
self._device = 'cuda'
elif use_cuda and not to.cuda.is_available():
warn(
'Tried to run on CUDA, but it is not available. Falling back to CPU.'
)
self._device = 'cpu'
# Register parameters
if learnable:
self.log_halfspan = nn.Parameter(to.Tensor(noise_dim),
requires_grad=True)
self.mean = nn.Parameter(
to.Tensor(noise_dim),
requires_grad=True) if train_mean else None
else:
self.log_halfspan = to.empty(noise_dim)
self.mean = None
# Initialize parameters
self.log_halfspan_init = to.log(
to.tensor(
halfspan_init, dtype=to.get_default_dtype())) if isinstance(
halfspan_init, (int, float)) else to.log(halfspan_init)
self.halfspan_min = to.tensor(halfspan_min) if isinstance(
halfspan_min, float) else halfspan_min
if not isinstance(self.log_halfspan_init, to.Tensor):
raise pyrado.TypeErr(given=self.log_halfspan_init,
expected_type=to.Tensor)
if not isinstance(self.halfspan_min, to.Tensor):
raise pyrado.TypeErr(given=self.halfspan_min,
expected_type=to.Tensor)
self.reset_expl_params()
self.to(self.device)
def test_sparse_true_divide(self, device, dtype):
dividend = torch.randn(5, device=device).to(dtype)
divisor = 2
dividend_sparse = dividend.to_sparse()
casting_result = dividend.to(torch.get_default_dtype()) / 2
self.assertEqual(casting_result, torch.true_divide(dividend_sparse, 2).to_dense())
def scale(self, waveform, factor=float(2**31)):
# scales a waveform by a factor
if not waveform.is_floating_point():
waveform = waveform.to(torch.get_default_dtype())
return waveform / factor
def test_cpp_frontend_module_python_inter_op(self):
extension = torch.utils.cpp_extension.load(
name="cpp_frontend_extension",
sources="cpp_extensions/cpp_frontend_extension.cpp",
verbose=True,
)
# Create a torch.nn.Module which uses the C++ module as a submodule.
class M(torch.nn.Module):
def __init__(self):
super(M, self).__init__()
self.x = torch.nn.Parameter(torch.tensor(1.0))
self.net = extension.Net(3, 5)
def forward(self, input):
return self.net.forward(input) + self.x
net = extension.Net(5, 2)
net.double()
net.to(torch.get_default_dtype())
self.assertEqual(str(net), "Net")
# Further embed the torch.nn.Module into a Sequential, and also add the
# C++ module as an element of the Sequential.
sequential = torch.nn.Sequential(M(), torch.nn.Tanh(), net,
torch.nn.Sigmoid())
input = torch.randn(2, 3)
# Try calling the module!
output = sequential.forward(input)
# The call operator is bound to forward too.
self.assertEqual(output, sequential(input))
self.assertEqual(list(output.shape), [2, 2])
# Do changes on the module hierarchy.
old_dtype = torch.get_default_dtype()
sequential.to(torch.float64)
sequential.to(torch.float32)
sequential.to(old_dtype)
self.assertEqual(sequential[2].parameters()[0].dtype, old_dtype)
# Make sure we can access these methods recursively.
self.assertEqual(len(list(sequential.parameters())),
len(net.parameters()) * 2 + 1)
self.assertEqual(len(list(sequential.named_parameters())),
len(net.named_parameters()) * 2 + 1)
self.assertEqual(len(list(sequential.buffers())),
len(net.buffers()) * 2)
self.assertEqual(len(list(sequential.modules())), 8)
# Test clone()
net2 = net.clone()
self.assertEqual(len(net.parameters()), len(net2.parameters()))
self.assertEqual(len(net.buffers()), len(net2.buffers()))
self.assertEqual(len(net.modules()), len(net2.modules()))
# Try differentiating through the whole module.
for parameter in net.parameters():
self.assertIsNone(parameter.grad)
output.sum().backward()
for parameter in net.parameters():
self.assertFalse(parameter.grad is None)
self.assertGreater(parameter.grad.sum(), 0)
# Try calling zero_grad()
net.zero_grad()
for p in net.parameters():
self.assertEqual(p.grad, torch.zeros_like(p))
# Test train(), eval(), training (a property)
self.assertTrue(net.training)
net.eval()
self.assertFalse(net.training)
net.train()
self.assertTrue(net.training)
net.eval()
# Try calling the additional methods we registered.
biased_input = torch.randn(4, 5)
output_before = net.forward(biased_input)
bias = net.get_bias().clone()
self.assertEqual(list(bias.shape), [2])
net.set_bias(bias + 1)
self.assertEqual(net.get_bias(), bias + 1)
output_after = net.forward(biased_input)
self.assertNotEqual(output_before, output_after)
# Try accessing parameters
self.assertEqual(len(net.parameters()), 2)
np = net.named_parameters()
self.assertEqual(len(np), 2)
self.assertIn("fc.weight", np)
self.assertIn("fc.bias", np)
self.assertEqual(len(net.buffers()), 1)
nb = net.named_buffers()
self.assertEqual(len(nb), 1)
self.assertIn("buf", nb)
self.assertEqual(nb[0][1], torch.eye(5))
def do_test_empty_full(self, dtypes, layout, device):
shape = torch.Size([2, 3])
def check_value(tensor, dtype, layout, device, value, requires_grad):
self.assertEqual(shape, tensor.shape)
self.assertIs(dtype, tensor.dtype)
self.assertIs(layout, tensor.layout)
self.assertEqual(tensor.requires_grad, requires_grad)
if tensor.is_cuda and device is not None:
self.assertEqual(device, tensor.device)
if value is not None:
fill = tensor.new(shape).fill_(value)
self.assertEqual(tensor, fill)
def get_int64_dtype(dtype):
module = '.'.join(str(dtype).split('.')[1:-1])
if not module:
return torch.int64
return operator.attrgetter(module)(torch).int64
default_dtype = torch.get_default_dtype()
check_value(torch.empty(shape), default_dtype, torch.strided, -1, None,
False)
check_value(torch.full(shape, -5), default_dtype, torch.strided, -1, None,
False)
for dtype in dtypes:
for rg in {dtype.is_floating_point, False}:
int64_dtype = get_int64_dtype(dtype)
v = torch.empty(shape,
dtype=dtype,
device=device,
layout=layout,
requires_grad=rg)
check_value(v, dtype, layout, device, None, rg)
out = v.new()
check_value(
torch.empty(shape,
out=out,
device=device,
layout=layout,
requires_grad=rg), dtype, layout, device, None, rg)
check_value(v.new_empty(shape), dtype, layout, device, None, False)
check_value(
v.new_empty(shape,
dtype=int64_dtype,
device=device,
requires_grad=False), int64_dtype, layout, device,
None, False)
check_value(torch.empty_like(v), dtype, layout, device, None,
False)
check_value(
torch.empty_like(v,
dtype=int64_dtype,
layout=layout,
device=device,
requires_grad=False), int64_dtype, layout,
device, None, False)
if dtype is not torch.float16 and layout != torch.sparse_coo:
fv = 3
v = torch.full(shape,
fv,
dtype=dtype,
layout=layout,
device=device,
requires_grad=rg)
check_value(v, dtype, layout, device, fv, rg)
check_value(v.new_full(shape, fv + 1), dtype, layout, device,
fv + 1, False)
out = v.new()
check_value(
torch.full(shape,
fv + 2,
out=out,
device=device,
layout=layout,
requires_grad=rg), dtype, layout, device,
fv + 2, rg)
check_value(
v.new_full(shape,
fv + 3,
dtype=int64_dtype,
device=device,
requires_grad=False), int64_dtype, layout,
device, fv + 3, False)
check_value(torch.full_like(v, fv + 4), dtype, layout, device,
fv + 4, False)
check_value(
torch.full_like(v,
fv + 5,
dtype=int64_dtype,
layout=layout,
device=device,
requires_grad=False), int64_dtype, layout,
device, fv + 5, False)
def func(x):
x = torch.as_tensor(x).type(torch.get_default_dtype())
ao = self.wf.ao(x, one_elec=True)
mo = self.wf.mo(self.wf.mo_scf(ao)).squeeze(1)
return mo[:, :self.mo_max_index].detach()
def __init__(self,
block: Union[BasicBlock, Bottleneck],
layers: Sequence[int],
n_fft: int = 256,
hop_length: Optional[int] = None,
win_length: Optional[int] = None,
window: Optional[str] = None,
normalized: bool = False,
onesided: bool = True,
spec_height: int = 224,
spec_width: int = 224,
num_classes: int = 1000,
pretrained: Union[bool, str] = False,
lock_pretrained: Optional[bool] = None):
super(_ESResNet, self).__init__(block=block,
layers=layers,
num_channels=3,
num_classes=num_classes)
self.num_classes = num_classes
self.fc = torch.nn.Identity()
self.classifier = torch.nn.Linear(in_features=512 * block.expansion,
out_features=self.num_classes)
if hop_length is None:
hop_length = int(np.floor(n_fft / 4))
if win_length is None:
win_length = n_fft
if window is None:
window = 'boxcar'
self.n_fft = n_fft
self.win_length = win_length
self.hop_length = hop_length
self.normalized = normalized
self.onesided = onesided
self.spec_height = spec_height
self.spec_width = spec_width
self.pretrained = pretrained
if pretrained:
err_msg = self.load_pretrained()
unlocked_weights = list()
for name, p in self.named_parameters():
if lock_pretrained and name not in err_msg:
p.requires_grad_(False)
else:
unlocked_weights.append(name)
print(f'Following weights are unlocked: {unlocked_weights}')
window_buffer: torch.Tensor = torch.from_numpy(
sps.get_window(window=window, Nx=win_length,
fftbins=True)).to(torch.get_default_dtype())
self.register_buffer('window', window_buffer)
self.log10_eps = 1e-18
def sgd(self, xt, verbose=False):
'''
sgd performs a single epoch of stochastic gradient descent on parameters
of f (Theta) and frequencies omega
Parameters
----------
xt : TYPE numpy.array
Temporal data whose first dimension is time.
verbose : TYPE boolean, optional
The default is False.
Returns
-------
TYPE float
Loss.
'''
batch_size = self.batch_size
T = xt.shape[0]
omega = nn.Parameter(self.omegas)
opt = optim.SGD(self.model_obj.parameters(), lr=3e-3)
opt_omega = optim.SGD([omega], lr=1e-7/T)
T = xt.shape[0]
t = torch.arange(T, device=self.device)
losses = []
for i in range(len(t)//batch_size):
ts = t[i*batch_size:(i+1)*batch_size]
o = torch.unsqueeze(omega, 0)
ts_ = torch.unsqueeze(ts,-1).type(torch.get_default_dtype()) + 1
xt_t = torch.tensor(xt[ts.cpu().numpy(),:], device=self.device)
wt = ts_*o
k = torch.cat([torch.cos(wt), torch.sin(wt)], -1)
loss = torch.mean(self.model_obj(k, xt_t))
opt.zero_grad()
opt_omega.zero_grad()
loss.backward()
opt.step()
opt_omega.step()
losses.append(loss.cpu().detach().numpy())
if verbose:
print('Setting to', 2*np.pi/omega)
self.omegas = omega.data
return np.mean(losses)
def __init__(self, dim, c=0.):
super().__init__(1)
self.register_buffer("dim", torch.as_tensor(dim, dtype=torch.int))
self.register_buffer(
"c", torch.as_tensor(c, dtype=torch.get_default_dtype()))
