def load(name, chains=None, model=None):
"""Load text database.
Parameters
----------
name : str
Path to root directory for text database
chains : list
Chains to load. If None, all chains are loaded.
model : Model
If None, the model is taken from the `with` context.
Returns
-------
ndarray.Trace instance
"""
chain_dirs = _get_chain_dirs(name)
if chains is None:
chains = list(chain_dirs.keys())
traces = []
for chain in chains:
chain_dir = chain_dirs[chain]
shape_file = os.path.join(chain_dir, 'shapes.json')
with open(shape_file, 'r') as sfh:
shapes = json.load(sfh)
samples = {}
for varname, shape in shapes.items():
var_file = os.path.join(chain_dir, varname + '.txt')
samples[varname] = np.loadtxt(var_file).reshape(shape)
trace = NDArray(model=model)
trace.samples = samples
trace.chain = chain
traces.append(trace)
return base.MultiTrace(traces)
def load(name, chains=None, model=None):
"""Load text database.
Parameters
----------
name : str
Path to root directory for text database
chains : list
Chains to load. If None, all chains are loaded.
model : Model
If None, the model is taken from the `with` context.
Returns
-------
ndarray.Trace instance
"""
chain_dirs = _get_chain_dirs(name)
if chains is None:
chains = list(chain_dirs.keys())
traces = []
for chain in chains:
chain_dir = chain_dirs[chain]
shape_file = os.path.join(chain_dir, 'shapes.json')
with open(shape_file, 'r') as sfh:
shapes = json.load(sfh)
samples = {}
for varname, shape in shapes.items():
var_file = os.path.join(chain_dir, varname + '.txt')
samples[varname] = np.loadtxt(var_file).reshape(shape)
trace = NDArray(model=model)
trace.samples = samples
trace.chain = chain
traces.append(trace)
return base.MultiTrace(traces)
def load(name,
chains=None,
model=None,
host='localhost',
port='50070',
user_name=None):
'''
Load text database
Parameters
----------
name : str
Path to root directory in HDFS for text database without a leading '/'
chains : list
Chains to load. If None, all chains are loaded
model : Model
If None, the model is taken from the 'with' context
host : str
The IP address or hostname of the HDFS namenode. By default,
it is 'localhost'
port : str
The port number for WebHDFS on the namenode. By default, it
is '50070'
user_name : str
WebHDFS user_name used for authentication. By default, it is
None
Returns
-------
ndarray.Trace instance
'''
hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
chain_dirs = _get_chain_dirs(name, hdfs)
if chains is None:
chains = list(chain_dirs.keys())
traces = []
for chain in chains:
chain_dir = chain_dirs[chain]
dir_path = os.path.join(name, chain_dir)
shape_file = os.path.join(dir_path, 'shapes.json')
shapes = json.load(StringIO.StringIO(hdfs.read_file(shape_file)))
samples = {}
for varname, shape in shapes.items():
var_file = os.path.join(dir_path, varname + '.txt')
samples[varname] = np.loadtxt(
StringIO.StringIO(str(
hdfs.read_file(var_file)))).reshape(shape)
trace = NDArray(model=model)
trace.samples = samples
trace.chain = chain
traces.append(trace)
return base.MultiTrace(traces)
def load(name, chains=None, model=None, host='localhost', port='50070', user_name=None):
'''
Load text database
Parameters
----------
name : str
Path to root directory in HDFS for text database without a leading '/'
chains : list
Chains to load. If None, all chains are loaded
model : Model
If None, the model is taken from the 'with' context
host : str
The IP address or hostname of the HDFS namenode. By default,
it is 'localhost'
port : str
The port number for WebHDFS on the namenode. By default, it
is '50070'
user_name : str
WebHDFS user_name used for authentication. By default, it is
None
Returns
-------
ndarray.Trace instance
'''
hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name)
chain_dirs = _get_chain_dirs(name, hdfs)
if chains is None:
chains = list(chain_dirs.keys())
traces = []
for chain in chains:
chain_dir = chain_dirs[chain]
dir_path = os.path.join(name, chain_dir)
shape_file = os.path.join(dir_path, 'shapes.json')
shapes = json.load(StringIO.StringIO(hdfs.read_file(shape_file)))
samples = {}
for varname, shape in shapes.items():
var_file = os.path.join(dir_path, varname + '.txt')
samples[varname] = np.loadtxt(StringIO.StringIO(str(hdfs.read_file(var_file)))).reshape(shape)
trace = NDArray(model=model)
trace.samples = samples
trace.chain = chain
traces.append(trace)
return base.MultiTrace(traces)
def _choose_backend(trace, chain, shortcuts=None, **kwds):
if isinstance(trace, BaseTrace):
return trace
if isinstance(trace, MultiTrace):
return trace._traces[chain]
if trace is None:
return NDArray(**kwds)
if shortcuts is None:
shortcuts = backends._shortcuts
try:
backend = shortcuts[trace]['backend']
name = shortcuts[trace]['name']
return backend(name, **kwds)
except TypeError:
return NDArray(vars=trace, **kwds)
except KeyError:
raise ValueError('Argument `trace` is invalid.')
def _posterior_to_trace(self, chain=0) -> NDArray:
"""Save results into a PyMC trace
This method should not be overwritten.
"""
lenght_pos = len(self.tempered_posterior)
varnames = [v.name for v in self.variables]
with self.model:
strace = NDArray(name=self.model.name)
strace.setup(lenght_pos, chain)
for i in range(lenght_pos):
value = []
size = 0
for varname in varnames:
shape, new_size = self.var_info[varname]
var_samples = self.tempered_posterior[i][size:size + new_size]
# Round discrete variable samples. The rounded values were the ones
# actually used in the logp evaluations (see logp_forw)
var = self.model[varname]
if var.dtype in discrete_types:
var_samples = np.round(var_samples).astype(var.dtype)
value.append(var_samples.reshape(shape))
size += new_size
strace.record(point={k: v for k, v in zip(varnames, value)})
return strace
def test_choose_chains(n_points, tune, expected_length, expected_n_traces):
with pm.Model() as model:
a = pm.Normal("a", mu=0, sigma=1)
trace_0 = NDArray(model)
trace_1 = NDArray(model)
trace_2 = NDArray(model)
trace_0.setup(n_points[0], 1)
trace_1.setup(n_points[1], 1)
trace_2.setup(n_points[2], 1)
for _ in range(n_points[0]):
trace_0.record({"a": 0})
for _ in range(n_points[1]):
trace_1.record({"a": 0})
for _ in range(n_points[2]):
trace_2.record({"a": 0})
traces, length = pm.sampling._choose_chains([trace_0, trace_1, trace_2], tune=tune)
assert length == expected_length
assert expected_n_traces == len(traces)
def test_sequential_backend(self):
with self.model:
backend = NDArray()
pm.sample(10, cores=1, chains=2, trace=backend)
