def hybrid_forward(self, F, pt, label, _alpha): if is_np_array(): softmax = F.npx.softmax pick = F.npx.pick ones_like = F.np.ones_like else: softmax = F.softmax pick = F.pick ones_like = F.ones_like if not self._from_logits: pt = softmax(pt, axis=self._axis) if not isinstance(_alpha, mx.base.numeric_types): _alpha = F.broadcast_mul(ones_like(pt),_alpha) _alpha = pick(_alpha, label, axis=self._axis, keepdims=True) if self._sparse_label: pt = pick(pt, label, axis=self._axis, keepdims=True) else: label = _reshape_like(F, label, pt) pt = (pt * label).sum(axis=self._axis, keepdims=True) loss = _apply_gamma_alpha(F, pt, self._gamma, _alpha) if is_np_array(): if F is mx.ndarray: return loss.mean(axis=tuple(range(1, loss.ndim))) else: return F.npx.batch_flatten(loss).mean(axis=1) else: return loss.mean(axis=self._batch_axis, exclude=True)
def __init__(self, offsets=None, rnd_type="uniform", factor_type="avg", magnitude=3, layout="NCHW"): super(SegmentedXavier, self).__init__(rnd_type, factor_type, magnitude) # check random type if self.rnd_type == "uniform": self.fn = mx.numpy.random.uniform if is_np_array() else mx.random.uniform elif self.rnd_type == "gaussian": self.fn = mx.numpy.random.normal if is_np_array() else mx.random.normal else: raise ValueError("Unknown random type") # check factor type if self.factor_type not in ["avg", "in", "out"]: raise ValueError("Incorrect factor type") self.offsets = offsets self.layout = layout
def _apply_gamma_alpha(F, pt, gamma=None, alpha=None): if gamma is not None: assert isinstance(gamma, numeric_types), "gamma must be a number" if is_np_array(): loss = -F.np.power(1-pt, gamma)*F.np.log(pt) else: loss = -F.power(1-pt, gamma)*F.log(pt) else: loss = -pt.log() if alpha is not None: if isinstance(alpha, numeric_types): loss = loss*alpha else: if is_np_array(): loss = loss*alpha else: loss = F.broadcast_mul(loss, alpha) return loss
def dict_batchify_fn(data): """dict batchify function""" if isinstance(data[0], mx.nd.NDArray): return _mx_np.stack(data) if is_np_array() else mx.nd.stack(*data) elif isinstance(data[0], dict): elem = data[0] return {key: dict_batchify_fn([d[key] for d in data]) for key in elem} raise RuntimeError('unknown datatype')
def _pad_arrs_to_max_length(arrs, pad_axis, pad_val, use_shared_mem, dtype, round_to=None): """Inner Implementation of the Pad batchify Parameters ---------- arrs : list pad_axis : int pad_val : number use_shared_mem : bool, default False dtype : round_to : int Returns ------- ret : NDArray original_length : NDArray """ if isinstance(arrs[0], mx.nd.NDArray): dtype = dtype or arrs[0].dtype arrs = [arr.asnumpy() for arr in arrs] elif not isinstance(arrs[0], np.ndarray): arrs = [np.asarray(ele) for ele in arrs] else: dtype = dtype or arrs[0].dtype original_length = [ele.shape[pad_axis] for ele in arrs] max_size = max(original_length) if round_to is not None: max_size = round_to * math.ceil(max_size / round_to) ret_shape = list(arrs[0].shape) ret_shape[pad_axis] = max_size ret_shape = (len(arrs), ) + tuple(ret_shape) ret = np.full(shape=ret_shape, fill_value=pad_val, dtype=dtype) for i, arr in enumerate(arrs): if arr.shape[pad_axis] == max_size: ret[i] = arr else: slices = [slice(None) for _ in range(arr.ndim)] slices[pad_axis] = slice(0, arr.shape[pad_axis]) if slices[pad_axis].start != slices[pad_axis].stop: slices = [slice(i, i + 1)] + slices ret[tuple(slices)] = arr ctx = mx.Context('cpu', 0) if use_shared_mem else mx.cpu() if is_np_array(): ret = mx.np.array(ret, ctx=ctx, dtype=dtype) else: ret = mx.nd.array(ret, ctx=ctx, dtype=dtype) return ret
def __init__(self,gamma=2, alpha=0.25, axis=-1, sparse_label=True, from_logits=False, weight=None, batch_axis=0, **kwargs): super(FocalLoss,self).__init__(weight, batch_axis, **kwargs) self._axis = axis self._sparse_label = sparse_label self._from_logits = from_logits self.epsilon = 1e-10 self._gamma = gamma self._alpha = alpha assert isinstance(self._alpha,(numeric_types,list,mx.nd.NDArray,np.ndarray)) if isinstance(self._alpha, list): if is_np_array(): self._alpha = np.array(self._alpha) else: self._alpha = mx.nd.array(self._alpha)
def _stack_arrs(arrs, use_shared_mem, dtype): if isinstance(arrs[0], mx.nd.NDArray): dtype = dtype or arrs[0].dtype if use_shared_mem: if is_np_array(): out = mx.np.empty((len(arrs), ) + arrs[0].shape, dtype=dtype, ctx=mx.Context('cpu_shared', 0)) return mx.np.stack(arrs, out=out) else: out = mx.nd.empty((len(arrs), ) + arrs[0].shape, dtype=dtype, ctx=mx.Context('cpu_shared', 0)) return mx.nd.stack(*arrs, out=out) else: if is_np_array(): return mx.np.stack(arrs) else: return mx.nd.stack(*arrs) else: out = np.asarray(arrs) dtype = dtype or out.dtype if use_shared_mem: if is_np_array(): return mx.np.array(out, ctx=mx.Context('cpu_shared', 0), dtype=dtype) else: return mx.nd.array(out, ctx=mx.Context('cpu_shared', 0), dtype=dtype) else: if is_np_array(): return mx.np.array(out, dtype=dtype) else: return mx.nd.array(out, dtype=dtype)
def export(self, path): """Export HybridBlock to json format that can be loaded by `gluon.SymbolBlock.imports`, `mxnet.mod.Module` or the C++ interface. .. note:: When there are only one input, it will have name `data`. When there Are more than one inputs, they will be named as `data0`, `data1`, etc. Parameters ---------- path : str Path to save model. Two files `path-symbol.json` and `path.params` will be created. Examples -------- >>> mynet.export('enas') >>> mynet_static = mx.gluon.nn.SymbolBlock.imports( "enas-symbol.json", ['data'], "enas.params") >>> x = mx.nd.random.uniform(shape=(1, 1, 28, 28)) >>> y = mynet_static(x) """ from mxnet import npx as _mx_npx from mxnet.util import is_np_array from mxnet import ndarray if not self._cached_graph: raise RuntimeError( "Please first call block.hybridize() and then run forward with " "this block at least once before calling export.") sym = self._cached_graph[1] sym.save('%s-symbol.json' % path, remove_amp_cast=True) arg_names = set(sym.list_arguments()) aux_names = set(sym.list_auxiliary_states()) arg_dict = {} for name, param in self.collect_params().items(): if name in arg_names: arg_dict['arg:%s' % name] = param._reduce() elif name in aux_names: arg_dict['aux:%s' % name] = param._reduce() else: pass save_fn = _mx_npx.save if is_np_array() else ndarray.save save_fn('%s.params' % (path), arg_dict)
def __init__(self, dataset, batch_size=None, shuffle=False, sampler=None, last_batch=None, batch_sampler=None, batchify_fn=None, num_workers=0, pin_memory=False, pin_device_id=0, prefetch=None, thread_pool=False, timeout=120, ctx_list=[cpu()]): self._dataset = dataset self._pin_memory = pin_memory self._pin_device_id = pin_device_id self._thread_pool = thread_pool self._timeout = timeout self._ctx_list = ctx_list assert timeout > 0, "timeout must be positive, given {}".format( timeout) # try to split the batch into shards for contexts num_ctx = len(ctx_list) bs_per_ctx = math.ceil(batch_size / float(num_ctx)) bs_list = [bs_per_ctx] * (num_ctx - 1) bs_list += [batch_size - sum(bs_list)] assert bs_list[-1] <= bs_per_ctx self._bs_list = np.cumsum([0] + bs_list) if pin_memory: warnings.warn('pin_memory not supported.') pin_memory = False if batch_sampler is None: if batch_size is None: raise ValueError("batch_size must be specified unless " \ "batch_sampler is specified") if sampler is None: if shuffle: sampler = _sampler.RandomSampler(len(dataset)) else: sampler = _sampler.SequentialSampler(len(dataset)) elif shuffle: raise ValueError( "shuffle must not be specified if sampler is specified") batch_sampler = _sampler.BatchSampler( sampler, batch_size, last_batch if last_batch else 'keep') elif batch_size is not None or shuffle or sampler is not None or \ last_batch is not None: raise ValueError("batch_size, shuffle, sampler and last_batch must " \ "not be specified if batch_sampler is specified.") self._batch_sampler = batch_sampler self._num_workers = num_workers if num_workers >= 0 else 0 if batchify_fn is None: self._batchify_fn = default_batchify_fn else: self._batchify_fn = batchify_fn self._iter = None self._pool = None if len(ctx_list) == 1 and ctx_list[0] == cpu() and num_workers > 0: self._iter = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, batch_sampler=batch_sampler, last_batch=last_batch, batchify_fn=batchify_fn, num_workers=num_workers, pin_memory=pin_memory, pin_device_id=pin_device_id, prefetch=prefetch, thread_pool=thread_pool, timeout=timeout) elif num_workers > 0: self._pool = ThreadPool(self._num_workers, initializer=_thread_worker_initializer, initargs=(is_np_shape(), is_np_array()))
def load_dict(self, param_dict, ctx=None, allow_missing=True, ignore_extra=True, cast_dtype=False, dtype_source="current"): """Load parameters from dict Parameters ---------- param_dict : dict Dictionary containing model parameters ctx : Context or list of Context Context(s) initialize loaded parameters on. allow_missing : bool, default False Whether to silently skip loading parameters not represented in the file. ignore_extra : bool, default False Whether to silently ignore parameters from the file that are not present in this dict. cast_dtype : bool, default False Cast the data type of the NDArray loaded from the checkpoint to the dtype provided by the Parameter if any dtype_source : str, default 'current' must be in {'current', 'saved'} Only valid if cast_dtype=True, specify the source of the dtype for casting the parameters """ if isinstance(param_dict.get('filename'), str): # pass from load_parameters filename = param_dict['filename'] param_dict = param_dict['params'] else: filename = None params = self.collect_params() error_str = "file: %s" % (filename) if filename else "param_dict" loaded = {k[4:] if k.startswith('arg:') or k.startswith('aux:') else k: v \ for k, v in param_dict.items()} if not allow_missing: params_inv = defaultdict(list) for k, v in params.items(): params_inv[v].append(k) for name, param in params.items(): assert any(p in loaded for p in params_inv[param]), \ "Parameter '%s' is missing in '%s', which contains parameters: %s. " \ "Set allow_missing=True to ignore missing parameters."%( name, error_str, _brief_print_list(loaded.keys())) if ctx is None: ctx = _context.current_context() for name in loaded: if not ignore_extra and name not in params: raise ValueError( "Parameter '%s' loaded from '%s' is not present in Dict, " \ "which contains parameters %s. Set ignore_extra=True to ignore. "%( name, error_str, _brief_print_list(params.keys()))) if name in params: param = loaded[name] if isinstance(param, np.ndarray): param = np.array(param) if is_np_array() else nd.array( param) params[name]._load_init(param, ctx, cast_dtype=cast_dtype, dtype_source=dtype_source)