def load(name, chains=None, model=None): """Load text database. Parameters ---------- name : str Path to root directory for text database chains : list Chains to load. If None, all chains are loaded. model : Model If None, the model is taken from the `with` context. Returns ------- ndarray.Trace instance """ chain_dirs = _get_chain_dirs(name) if chains is None: chains = list(chain_dirs.keys()) traces = [] for chain in chains: chain_dir = chain_dirs[chain] shape_file = os.path.join(chain_dir, 'shapes.json') with open(shape_file, 'r') as sfh: shapes = json.load(sfh) samples = {} for varname, shape in shapes.items(): var_file = os.path.join(chain_dir, varname + '.txt') samples[varname] = np.loadtxt(var_file).reshape(shape) trace = NDArray(model=model) trace.samples = samples trace.chain = chain traces.append(trace) return base.MultiTrace(traces)
def load(name, chains=None, model=None, host='localhost', port='50070', user_name=None): ''' Load text database Parameters ---------- name : str Path to root directory in HDFS for text database without a leading '/' chains : list Chains to load. If None, all chains are loaded model : Model If None, the model is taken from the 'with' context host : str The IP address or hostname of the HDFS namenode. By default, it is 'localhost' port : str The port number for WebHDFS on the namenode. By default, it is '50070' user_name : str WebHDFS user_name used for authentication. By default, it is None Returns ------- ndarray.Trace instance ''' hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name) chain_dirs = _get_chain_dirs(name, hdfs) if chains is None: chains = list(chain_dirs.keys()) traces = [] for chain in chains: chain_dir = chain_dirs[chain] dir_path = os.path.join(name, chain_dir) shape_file = os.path.join(dir_path, 'shapes.json') shapes = json.load(StringIO.StringIO(hdfs.read_file(shape_file))) samples = {} for varname, shape in shapes.items(): var_file = os.path.join(dir_path, varname + '.txt') samples[varname] = np.loadtxt( StringIO.StringIO(str( hdfs.read_file(var_file)))).reshape(shape) trace = NDArray(model=model) trace.samples = samples trace.chain = chain traces.append(trace) return base.MultiTrace(traces)
def load(name, chains=None, model=None, host='localhost', port='50070', user_name=None): ''' Load text database Parameters ---------- name : str Path to root directory in HDFS for text database without a leading '/' chains : list Chains to load. If None, all chains are loaded model : Model If None, the model is taken from the 'with' context host : str The IP address or hostname of the HDFS namenode. By default, it is 'localhost' port : str The port number for WebHDFS on the namenode. By default, it is '50070' user_name : str WebHDFS user_name used for authentication. By default, it is None Returns ------- ndarray.Trace instance ''' hdfs = PyWebHdfsClient(host=host, port=port, user_name=user_name) chain_dirs = _get_chain_dirs(name, hdfs) if chains is None: chains = list(chain_dirs.keys()) traces = [] for chain in chains: chain_dir = chain_dirs[chain] dir_path = os.path.join(name, chain_dir) shape_file = os.path.join(dir_path, 'shapes.json') shapes = json.load(StringIO.StringIO(hdfs.read_file(shape_file))) samples = {} for varname, shape in shapes.items(): var_file = os.path.join(dir_path, varname + '.txt') samples[varname] = np.loadtxt(StringIO.StringIO(str(hdfs.read_file(var_file)))).reshape(shape) trace = NDArray(model=model) trace.samples = samples trace.chain = chain traces.append(trace) return base.MultiTrace(traces)
def _choose_backend(trace, chain, shortcuts=None, **kwds): if isinstance(trace, BaseTrace): return trace if isinstance(trace, MultiTrace): return trace._traces[chain] if trace is None: return NDArray(**kwds) if shortcuts is None: shortcuts = backends._shortcuts try: backend = shortcuts[trace]['backend'] name = shortcuts[trace]['name'] return backend(name, **kwds) except TypeError: return NDArray(vars=trace, **kwds) except KeyError: raise ValueError('Argument `trace` is invalid.')
def _posterior_to_trace(self, chain=0) -> NDArray: """Save results into a PyMC trace This method should not be overwritten. """ lenght_pos = len(self.tempered_posterior) varnames = [v.name for v in self.variables] with self.model: strace = NDArray(name=self.model.name) strace.setup(lenght_pos, chain) for i in range(lenght_pos): value = [] size = 0 for varname in varnames: shape, new_size = self.var_info[varname] var_samples = self.tempered_posterior[i][size:size + new_size] # Round discrete variable samples. The rounded values were the ones # actually used in the logp evaluations (see logp_forw) var = self.model[varname] if var.dtype in discrete_types: var_samples = np.round(var_samples).astype(var.dtype) value.append(var_samples.reshape(shape)) size += new_size strace.record(point={k: v for k, v in zip(varnames, value)}) return strace
def test_choose_chains(n_points, tune, expected_length, expected_n_traces): with pm.Model() as model: a = pm.Normal("a", mu=0, sigma=1) trace_0 = NDArray(model) trace_1 = NDArray(model) trace_2 = NDArray(model) trace_0.setup(n_points[0], 1) trace_1.setup(n_points[1], 1) trace_2.setup(n_points[2], 1) for _ in range(n_points[0]): trace_0.record({"a": 0}) for _ in range(n_points[1]): trace_1.record({"a": 0}) for _ in range(n_points[2]): trace_2.record({"a": 0}) traces, length = pm.sampling._choose_chains([trace_0, trace_1, trace_2], tune=tune) assert length == expected_length assert expected_n_traces == len(traces)
def test_sequential_backend(self): with self.model: backend = NDArray() pm.sample(10, cores=1, chains=2, trace=backend)