def load(fname):
"""Loads an array from file.
See more details in ``save``.
Parameters
----------
fname : str
The filename.
Returns
-------
list of NDArray or dict of str to NDArray
Loaded data.
"""
if not isinstance(fname, string_types):
raise TypeError('fname required to be a string')
out_size = mx_uint()
out_name_size = mx_uint()
handles = ctypes.POINTER(NDArrayHandle)()
names = ctypes.POINTER(ctypes.c_char_p)()
check_call(
_LIB.MXNDArrayLoad(c_str(fname), ctypes.byref(out_size),
ctypes.byref(handles), ctypes.byref(out_name_size),
ctypes.byref(names)))
if out_name_size.value == 0:
return [
_ndarray_cls(NDArrayHandle(handles[i]))
for i in range(out_size.value)
]
else:
assert out_name_size.value == out_size.value
return dict((py_str(names[i]), _ndarray_cls(NDArrayHandle(handles[i])))
for i in range(out_size.value))
def _init_symbol_module(root_namespace):
"""List and add all the atomic symbol functions to current module."""
plist = ctypes.POINTER(ctypes.c_char_p)()
size = ctypes.c_uint()
check_call(_LIB.MXListAllOpNames(ctypes.byref(size),
ctypes.byref(plist)))
op_names = []
for i in range(size.value):
op_names.append(py_str(plist[i]))
module_obj = _sys.modules["%s.symbol" % root_namespace]
module_sparse = _sys.modules["%s.symbol.sparse" % root_namespace]
module_internal = _sys.modules["%s.symbol._internal" % root_namespace]
module_contrib = _sys.modules["%s.contrib.symbol" % root_namespace]
for name in op_names:
hdl = OpHandle()
check_call(_LIB.NNGetOpHandle(c_str(name), ctypes.byref(hdl)))
function = _make_atomic_symbol_function(hdl, name)
if function.__name__.startswith('_contrib_'):
function.__name__ = function.__name__[9:]
function.__module__ = 'mxnet.contrib.symbol'
setattr(module_contrib, function.__name__, function)
elif function.__name__.startswith('_'):
setattr(module_internal, function.__name__, function)
else:
setattr(module_obj, function.__name__, function)
# register sparse ops under mxnet.symbol.sparse
if function.__name__.startswith('_sparse_'):
function.__name__ = function.__name__[8:]
function.__module__ = 'mxnet.symbol.sparse'
setattr(module_sparse, function.__name__, function)
def collect(self, name, arr):
"""Callback function for collecting min and max values from an NDArray."""
name = py_str(name)
if name not in self.include_layer:
return
handle = ctypes.cast(arr, NDArrayHandle)
arr = mx.ndarray.NDArray(handle, writable=False).asnumpy() # pylint: disable=no-member
if name in self.tensor_dict.keys():
self.tensor_dict[name].append(arr)
else:
self.tensor_dict[name] = [arr]
def collect(self, name, arr):
"""Callback function for collecting layer output NDArrays."""
name = py_str(name)
if self.include_layer is not None and not self.include_layer(name):
return
handle = ctypes.cast(arr, NDArrayHandle)
arr = NDArray(handle, writable=False).copyto(cpu())
if self.logger is not None:
self.logger.info("Collecting layer %s output of shape %s" %
(name, arr.shape))
if name in self.nd_dict:
self.nd_dict[name].append(arr)
else:
self.nd_dict[name] = [arr]
def _get_op_arguments(op_handle):
"""Given operator name and handle, fetch operator arguments - number of arguments,
argument names, argument types.
Parameters
----------
op_handle: OpHandle
Handle for the operator
Returns
-------
(narg, arg_names, arg_types)
"""
real_name = ctypes.c_char_p()
desc = ctypes.c_char_p()
num_args = mx_uint()
arg_names = ctypes.POINTER(ctypes.c_char_p)()
arg_types = ctypes.POINTER(ctypes.c_char_p)()
arg_descs = ctypes.POINTER(ctypes.c_char_p)()
key_var_num_args = ctypes.c_char_p()
ret_type = ctypes.c_char_p()
check_call(
_LIB.MXSymbolGetAtomicSymbolInfo(op_handle, ctypes.byref(real_name),
ctypes.byref(desc),
ctypes.byref(num_args),
ctypes.byref(arg_names),
ctypes.byref(arg_types),
ctypes.byref(arg_descs),
ctypes.byref(key_var_num_args),
ctypes.byref(ret_type)))
narg = int(num_args.value)
arg_names = [py_str(arg_names[i]) for i in range(narg)]
arg_types = [py_str(arg_types[i]) for i in range(narg)]
return narg, arg_names, arg_types
def _get_all_registered_ops():
"""Get all registered MXNet operator names.
Returns
-------
["operator_name"]
"""
plist = ctypes.POINTER(ctypes.c_char_p)()
size = ctypes.c_uint()
check_call(_LIB.MXListAllOpNames(ctypes.byref(size),
ctypes.byref(plist)))
mx_registered_operator_names = [py_str(plist[i]) for i in range(size.value)]
return mx_registered_operator_names
def collect(self, name, arr):
"""Callback function for collecting min and max values from an NDArray."""
name = py_str(name)
if self.include_layer is not None and not self.include_layer(name):
return
handle = ctypes.cast(arr, NDArrayHandle)
arr = NDArray(handle, writable=False)
min_range = ndarray.min(arr).asscalar()
max_range = ndarray.max(arr).asscalar()
if name in self.min_max_dict:
cur_min_max = self.min_max_dict[name]
self.min_max_dict[name] = (min(cur_min_max[0], min_range),
max(cur_min_max[1], max_range))
else:
self.min_max_dict[name] = (min_range, max_range)
if self.logger is not None:
self.logger.info("Collecting layer %s min_range=%f, max_range=%f" %
(name, min_range, max_range))
def str_updater(key, recv, local):
"""use updater: += with str keys"""
if isinstance(key, bytes):
key = py_str(key)
assert(isinstance(key, str))
local += recv
def _make_atomic_symbol_function(handle, name):
"""Create an atomic symbol function by handle and function name."""
real_name = ctypes.c_char_p()
desc = ctypes.c_char_p()
num_args = mx_uint()
arg_names = ctypes.POINTER(ctypes.c_char_p)()
arg_types = ctypes.POINTER(ctypes.c_char_p)()
arg_descs = ctypes.POINTER(ctypes.c_char_p)()
key_var_num_args = ctypes.c_char_p()
ret_type = ctypes.c_char_p()
check_call(_LIB.MXSymbolGetAtomicSymbolInfo(
handle, ctypes.byref(real_name), ctypes.byref(desc),
ctypes.byref(num_args),
ctypes.byref(arg_names),
ctypes.byref(arg_types),
ctypes.byref(arg_descs),
ctypes.byref(key_var_num_args),
ctypes.byref(ret_type)))
narg = int(num_args.value)
arg_names = [py_str(arg_names[i]) for i in range(narg)]
arg_types = [py_str(arg_types[i]) for i in range(narg)]
func_name = name
key_var_num_args = py_str(key_var_num_args.value)
ret_type = py_str(ret_type.value) if ret_type.value is not None else ''
doc_str = _build_doc(func_name,
py_str(desc.value),
arg_names,
arg_types,
[py_str(arg_descs[i]) for i in range(narg)],
key_var_num_args,
ret_type)
dtype_name = None
arr_name = None
ndsignature = []
signature = []
ndarg_names = []
kwarg_names = []
for i in range(narg):
name, atype = arg_names[i], arg_types[i]
if name == 'dtype':
dtype_name = name
signature.append('%s=_Null'%name)
elif atype.startswith('NDArray') or atype.startswith('Symbol'):
assert not arr_name, \
"Op can only have one argument with variable " \
"size and it must be the last argument."
if atype.endswith('[]'):
ndsignature.append('*%s'%name)
arr_name = name
else:
ndsignature.append('%s=None'%name)
ndarg_names.append(name)
else:
signature.append('%s=_Null'%name)
kwarg_names.append(name)
#signature.append('is_train=False')
signature.append('name=None')
signature.append('attr=None')
signature.append('out=None')
signature.append('**kwargs')
signature = ndsignature + signature
code = []
if arr_name:
code.append("""
def %s(*%s, **kwargs):"""%(func_name, arr_name))
code.append("""
sym_args = []
for i in {}:
assert isinstance(i, SymbolBase), \\
"Positional arguments must be Symbol instances, " \\
"but got %s"%str(i)
sym_args.append(i)""".format(arr_name))
if dtype_name is not None:
code.append("""
if '%s' in kwargs:
kwargs['%s'] = _numpy.dtype(kwargs['%s']).name"""%(
dtype_name, dtype_name, dtype_name))
code.append("""
attr = kwargs.pop('attr', None)
kwargs.update(AttrScope.current.get(attr))
name = kwargs.pop('name', None)
name = NameManager.current.get(name, '%s')
_ = kwargs.pop('out', None)
keys = []
vals = []
sym_kwargs = dict()
for k, v in kwargs.items():
if isinstance(v, SymbolBase):
sym_kwargs[k] = v
else:
keys.append(k)
vals.append(v)"""%(func_name.lower()))
if key_var_num_args:
code.append("""
if '%s' not in kwargs:
keys.append('%s')
vals.append(len(sym_args) + len(sym_kwargs))"""%(
key_var_num_args, key_var_num_args))
code.append("""
return _symbol_creator(%d, sym_args, sym_kwargs, keys, vals, name)"""%(
handle.value))
else:
code.append("""
def %s(%s):
kwargs.update(AttrScope.current.get(attr))
sym_kwargs = dict()
keys = []
vals = []"""%(func_name, ', '.join(signature)))
code.append("""
for k, v in kwargs.items():
if isinstance(v, SymbolBase):
sym_kwargs[k] = v
else:
keys.append(k)
vals.append(v)""")
# NDArray args
for name in ndarg_names: # pylint: disable=redefined-argument-from-local
code.append("""
if {name} is not None:
assert isinstance({name}, SymbolBase), \\
"Argument {name} must be Symbol instances, but got %s"%str({name})
sym_kwargs['{name}'] = {name}""".format(name=name))
# kwargs
for name in kwarg_names: # pylint: disable=redefined-argument-from-local
code.append("""
if %s is not _Null:
keys.append('%s')
vals.append(%s)"""%(name, name, name))
# dtype
if dtype_name is not None:
code.append("""
if %s is not _Null:
keys.append('%s')
vals.append(_numpy.dtype(%s).name)"""%(dtype_name, dtype_name, dtype_name))
code.append("""
name = NameManager.current.get(name, '%s')
return _symbol_creator(%d, None, sym_kwargs, keys, vals, name)"""%(
func_name.lower(), handle.value))
local = {}
exec(''.join(code), None, local) # pylint: disable=exec-used
symbol_function = local[func_name]
symbol_function.__name__ = func_name
symbol_function.__doc__ = doc_str
symbol_function.__module__ = 'mxnet.symbol'
return symbol_function
def stat_helper(name, array):
array = ctypes.cast(array, NDArrayHandle)
array = NDArray(array, writable=False)
if not self.activated or not self.re_prog.match(py_str(name)):
return
self.queue.append((self.step, py_str(name), stat(array)))
