def hybrid_forward(self, F, pt, label, _alpha):
if is_np_array():
softmax = F.npx.softmax
pick = F.npx.pick
ones_like = F.np.ones_like
else:
softmax = F.softmax
pick = F.pick
ones_like = F.ones_like
if not self._from_logits:
pt = softmax(pt, axis=self._axis)
if not isinstance(_alpha, mx.base.numeric_types):
_alpha = F.broadcast_mul(ones_like(pt),_alpha)
_alpha = pick(_alpha, label, axis=self._axis, keepdims=True)
if self._sparse_label:
pt = pick(pt, label, axis=self._axis, keepdims=True)
else:
label = _reshape_like(F, label, pt)
pt = (pt * label).sum(axis=self._axis, keepdims=True)
loss = _apply_gamma_alpha(F, pt, self._gamma, _alpha)
if is_np_array():
if F is mx.ndarray:
return loss.mean(axis=tuple(range(1, loss.ndim)))
else:
return F.npx.batch_flatten(loss).mean(axis=1)
else:
return loss.mean(axis=self._batch_axis, exclude=True)
def __init__(self, offsets=None, rnd_type="uniform", factor_type="avg", magnitude=3, layout="NCHW"):
super(SegmentedXavier, self).__init__(rnd_type, factor_type, magnitude)
# check random type
if self.rnd_type == "uniform":
self.fn = mx.numpy.random.uniform if is_np_array() else mx.random.uniform
elif self.rnd_type == "gaussian":
self.fn = mx.numpy.random.normal if is_np_array() else mx.random.normal
else:
raise ValueError("Unknown random type")
# check factor type
if self.factor_type not in ["avg", "in", "out"]:
raise ValueError("Incorrect factor type")
self.offsets = offsets
self.layout = layout
def _apply_gamma_alpha(F, pt, gamma=None, alpha=None):
if gamma is not None:
assert isinstance(gamma, numeric_types), "gamma must be a number"
if is_np_array():
loss = -F.np.power(1-pt, gamma)*F.np.log(pt)
else:
loss = -F.power(1-pt, gamma)*F.log(pt)
else:
loss = -pt.log()
if alpha is not None:
if isinstance(alpha, numeric_types):
loss = loss*alpha
else:
if is_np_array():
loss = loss*alpha
else:
loss = F.broadcast_mul(loss, alpha)
return loss
def dict_batchify_fn(data):
"""dict batchify function"""
if isinstance(data[0], mx.nd.NDArray):
return _mx_np.stack(data) if is_np_array() else mx.nd.stack(*data)
elif isinstance(data[0], dict):
elem = data[0]
return {key: dict_batchify_fn([d[key] for d in data]) for key in elem}
raise RuntimeError('unknown datatype')
def _pad_arrs_to_max_length(arrs,
pad_axis,
pad_val,
use_shared_mem,
dtype,
round_to=None):
"""Inner Implementation of the Pad batchify
Parameters
----------
arrs : list
pad_axis : int
pad_val : number
use_shared_mem : bool, default False
dtype :
round_to : int
Returns
-------
ret : NDArray
original_length : NDArray
"""
if isinstance(arrs[0], mx.nd.NDArray):
dtype = dtype or arrs[0].dtype
arrs = [arr.asnumpy() for arr in arrs]
elif not isinstance(arrs[0], np.ndarray):
arrs = [np.asarray(ele) for ele in arrs]
else:
dtype = dtype or arrs[0].dtype
original_length = [ele.shape[pad_axis] for ele in arrs]
max_size = max(original_length)
if round_to is not None:
max_size = round_to * math.ceil(max_size / round_to)
ret_shape = list(arrs[0].shape)
ret_shape[pad_axis] = max_size
ret_shape = (len(arrs), ) + tuple(ret_shape)
ret = np.full(shape=ret_shape, fill_value=pad_val, dtype=dtype)
for i, arr in enumerate(arrs):
if arr.shape[pad_axis] == max_size:
ret[i] = arr
else:
slices = [slice(None) for _ in range(arr.ndim)]
slices[pad_axis] = slice(0, arr.shape[pad_axis])
if slices[pad_axis].start != slices[pad_axis].stop:
slices = [slice(i, i + 1)] + slices
ret[tuple(slices)] = arr
ctx = mx.Context('cpu', 0) if use_shared_mem else mx.cpu()
if is_np_array():
ret = mx.np.array(ret, ctx=ctx, dtype=dtype)
else:
ret = mx.nd.array(ret, ctx=ctx, dtype=dtype)
return ret
def __init__(self,gamma=2, alpha=0.25, axis=-1, sparse_label=True,
from_logits=False, weight=None, batch_axis=0, **kwargs):
super(FocalLoss,self).__init__(weight, batch_axis, **kwargs)
self._axis = axis
self._sparse_label = sparse_label
self._from_logits = from_logits
self.epsilon = 1e-10
self._gamma = gamma
self._alpha = alpha
assert isinstance(self._alpha,(numeric_types,list,mx.nd.NDArray,np.ndarray))
if isinstance(self._alpha, list):
if is_np_array():
self._alpha = np.array(self._alpha)
else:
self._alpha = mx.nd.array(self._alpha)
def _stack_arrs(arrs, use_shared_mem, dtype):
if isinstance(arrs[0], mx.nd.NDArray):
dtype = dtype or arrs[0].dtype
if use_shared_mem:
if is_np_array():
out = mx.np.empty((len(arrs), ) + arrs[0].shape,
dtype=dtype,
ctx=mx.Context('cpu_shared', 0))
return mx.np.stack(arrs, out=out)
else:
out = mx.nd.empty((len(arrs), ) + arrs[0].shape,
dtype=dtype,
ctx=mx.Context('cpu_shared', 0))
return mx.nd.stack(*arrs, out=out)
else:
if is_np_array():
return mx.np.stack(arrs)
else:
return mx.nd.stack(*arrs)
else:
out = np.asarray(arrs)
dtype = dtype or out.dtype
if use_shared_mem:
if is_np_array():
return mx.np.array(out,
ctx=mx.Context('cpu_shared', 0),
dtype=dtype)
else:
return mx.nd.array(out,
ctx=mx.Context('cpu_shared', 0),
dtype=dtype)
else:
if is_np_array():
return mx.np.array(out, dtype=dtype)
else:
return mx.nd.array(out, dtype=dtype)
def export(self, path):
"""Export HybridBlock to json format that can be loaded by
`gluon.SymbolBlock.imports`, `mxnet.mod.Module` or the C++ interface.
.. note:: When there are only one input, it will have name `data`. When there
Are more than one inputs, they will be named as `data0`, `data1`, etc.
Parameters
----------
path : str
Path to save model. Two files `path-symbol.json` and `path.params`
will be created.
Examples
--------
>>> mynet.export('enas')
>>> mynet_static = mx.gluon.nn.SymbolBlock.imports(
"enas-symbol.json", ['data'], "enas.params")
>>> x = mx.nd.random.uniform(shape=(1, 1, 28, 28))
>>> y = mynet_static(x)
"""
from mxnet import npx as _mx_npx
from mxnet.util import is_np_array
from mxnet import ndarray
if not self._cached_graph:
raise RuntimeError(
"Please first call block.hybridize() and then run forward with "
"this block at least once before calling export.")
sym = self._cached_graph[1]
sym.save('%s-symbol.json' % path, remove_amp_cast=True)
arg_names = set(sym.list_arguments())
aux_names = set(sym.list_auxiliary_states())
arg_dict = {}
for name, param in self.collect_params().items():
if name in arg_names:
arg_dict['arg:%s' % name] = param._reduce()
elif name in aux_names:
arg_dict['aux:%s' % name] = param._reduce()
else:
pass
save_fn = _mx_npx.save if is_np_array() else ndarray.save
save_fn('%s.params' % (path), arg_dict)
def __init__(self,
dataset,
batch_size=None,
shuffle=False,
sampler=None,
last_batch=None,
batch_sampler=None,
batchify_fn=None,
num_workers=0,
pin_memory=False,
pin_device_id=0,
prefetch=None,
thread_pool=False,
timeout=120,
ctx_list=[cpu()]):
self._dataset = dataset
self._pin_memory = pin_memory
self._pin_device_id = pin_device_id
self._thread_pool = thread_pool
self._timeout = timeout
self._ctx_list = ctx_list
assert timeout > 0, "timeout must be positive, given {}".format(
timeout)
# try to split the batch into shards for contexts
num_ctx = len(ctx_list)
bs_per_ctx = math.ceil(batch_size / float(num_ctx))
bs_list = [bs_per_ctx] * (num_ctx - 1)
bs_list += [batch_size - sum(bs_list)]
assert bs_list[-1] <= bs_per_ctx
self._bs_list = np.cumsum([0] + bs_list)
if pin_memory:
warnings.warn('pin_memory not supported.')
pin_memory = False
if batch_sampler is None:
if batch_size is None:
raise ValueError("batch_size must be specified unless " \
"batch_sampler is specified")
if sampler is None:
if shuffle:
sampler = _sampler.RandomSampler(len(dataset))
else:
sampler = _sampler.SequentialSampler(len(dataset))
elif shuffle:
raise ValueError(
"shuffle must not be specified if sampler is specified")
batch_sampler = _sampler.BatchSampler(
sampler, batch_size, last_batch if last_batch else 'keep')
elif batch_size is not None or shuffle or sampler is not None or \
last_batch is not None:
raise ValueError("batch_size, shuffle, sampler and last_batch must " \
"not be specified if batch_sampler is specified.")
self._batch_sampler = batch_sampler
self._num_workers = num_workers if num_workers >= 0 else 0
if batchify_fn is None:
self._batchify_fn = default_batchify_fn
else:
self._batchify_fn = batchify_fn
self._iter = None
self._pool = None
if len(ctx_list) == 1 and ctx_list[0] == cpu() and num_workers > 0:
self._iter = DataLoader(dataset,
batch_size=batch_size,
shuffle=shuffle,
batch_sampler=batch_sampler,
last_batch=last_batch,
batchify_fn=batchify_fn,
num_workers=num_workers,
pin_memory=pin_memory,
pin_device_id=pin_device_id,
prefetch=prefetch,
thread_pool=thread_pool,
timeout=timeout)
elif num_workers > 0:
self._pool = ThreadPool(self._num_workers,
initializer=_thread_worker_initializer,
initargs=(is_np_shape(), is_np_array()))
def load_dict(self,
param_dict,
ctx=None,
allow_missing=True,
ignore_extra=True,
cast_dtype=False,
dtype_source="current"):
"""Load parameters from dict
Parameters
----------
param_dict : dict
Dictionary containing model parameters
ctx : Context or list of Context
Context(s) initialize loaded parameters on.
allow_missing : bool, default False
Whether to silently skip loading parameters not represented in the file.
ignore_extra : bool, default False
Whether to silently ignore parameters from the file that are not
present in this dict.
cast_dtype : bool, default False
Cast the data type of the NDArray loaded from the checkpoint to the dtype
provided by the Parameter if any
dtype_source : str, default 'current'
must be in {'current', 'saved'}
Only valid if cast_dtype=True, specify the source of the dtype for casting
the parameters
"""
if isinstance(param_dict.get('filename'), str):
# pass from load_parameters
filename = param_dict['filename']
param_dict = param_dict['params']
else:
filename = None
params = self.collect_params()
error_str = "file: %s" % (filename) if filename else "param_dict"
loaded = {k[4:] if k.startswith('arg:') or k.startswith('aux:') else k: v \
for k, v in param_dict.items()}
if not allow_missing:
params_inv = defaultdict(list)
for k, v in params.items():
params_inv[v].append(k)
for name, param in params.items():
assert any(p in loaded for p in params_inv[param]), \
"Parameter '%s' is missing in '%s', which contains parameters: %s. " \
"Set allow_missing=True to ignore missing parameters."%(
name, error_str, _brief_print_list(loaded.keys()))
if ctx is None:
ctx = _context.current_context()
for name in loaded:
if not ignore_extra and name not in params:
raise ValueError(
"Parameter '%s' loaded from '%s' is not present in Dict, " \
"which contains parameters %s. Set ignore_extra=True to ignore. "%(
name, error_str, _brief_print_list(params.keys())))
if name in params:
param = loaded[name]
if isinstance(param, np.ndarray):
param = np.array(param) if is_np_array() else nd.array(
param)
params[name]._load_init(param,
ctx,
cast_dtype=cast_dtype,
dtype_source=dtype_source)