def test_find_MAP_discrete(): tol = 2.0**-11 alpha = 4 beta = 4 n = 20 yes = 15 with Model() as model: p = Beta('p', alpha, beta, transform=None) ss = Binomial('ss', n=n, p=p, transform=None) s = Binomial('s', n=n, p=p, observed=yes) map_est1 = starting.find_MAP() map_est2 = starting.find_MAP(vars=model.vars) close_to(map_est1['p'], 0.6086956533498806, tol) close_to(map_est2['p'], 0.695642178810167, tol) assert map_est2['ss'] == 14
def test_find_MAP_discrete(): tol = 2.0**-11 alpha = 4 beta = 4 n = 20 yes = 15 with Model() as model: p = Beta("p", alpha, beta) Binomial("ss", n=n, p=p) Binomial("s", n=n, p=p, observed=yes) map_est1 = starting.find_MAP() map_est2 = starting.find_MAP(vars=model.vars) close_to(map_est1["p"], 0.6086956533498806, tol) close_to(map_est2["p"], 0.695642178810167, tol) assert map_est2["ss"] == 14
def hmetad_groupLevel(data: dict, sample_model: bool = True, **kwargs): """Compute hierachical meta-d' at the subject level. This is an internal function. The group level model must be called using :py:func:`metadPy.hierarchical.hmetad`. Parameters ---------- data : dict Response data. sample_model : boolean If `False`, only the model is returned without sampling. **kwargs : keyword arguments All keyword arguments are passed to `func::pymc3.sampling.sample`. Returns ------- model : :py:class:`pymc3.Model` instance The pymc3 model. Encapsulates the variables and likelihood factors. trace : :py:class:`pymc3.backends.base.MultiTrace` or :py:class:`arviz.InferenceData` A `MultiTrace` or `ArviZ InferenceData` object that contains the samples. References ---------- .. [#] Fleming, S.M. (2017) HMeta-d: hierarchical Bayesian estimation of metacognitive efficiency from confidence ratings, Neuroscience of Consciousness, 3(1) nix007, https://doi.org/10.1093/nc/nix007 """ nSubj = data["nSubj"] hits = data["hits"] falsealarms = data["falsealarms"] s = data["s"] n = data["n"] counts = data["counts"] nRatings = data["nRatings"] Tol = data["Tol"] cr = data["cr"] m = data["m"] with Model() as model: # hyperpriors on d, c and c2 mu_c1 = Normal( "mu_c1", mu=0, tau=0.01, shape=(1), testval=np.random.rand() * 0.1 ) mu_c2 = Normal( "mu_c2", mu=0, tau=0.01, shape=(1, 1), testval=np.random.rand() * 0.1 ) mu_d1 = Normal( "mu_d1", mu=0, tau=0.01, shape=(1), testval=np.random.rand() * 0.1 ) sigma_c1 = HalfNormal( "sigma_c1", tau=0.01, shape=(1), testval=np.random.rand() * 0.1 ) sigma_c2 = HalfNormal( "sigma_c2", tau=0.01, shape=(1, 1), testval=np.random.rand() * 0.1 ) sigma_d1 = HalfNormal( "sigma_d1", tau=0.01, shape=(1), testval=np.random.rand() * 0.1 ) # Type 1 priors c1_tilde = Normal("c1_tilde", mu=0, sigma=1, shape=(nSubj, 1)) c1 = Deterministic("c1", mu_c1 + sigma_c1 * c1_tilde) d1_tilde = Normal("d1_tilde", mu=0, sigma=1, shape=(nSubj, 1)) d1 = Deterministic("d1", mu_d1 + sigma_d1 * d1_tilde) # TYPE 1 SDT BINOMIAL MODEL h = cumulative_normal(d1 / 2 - c1) f = cumulative_normal(-d1 / 2 - c1) H = Binomial("H", n=s, p=h, observed=hits) FA = Binomial("FA", n=n, p=f, observed=falsealarms) # Hyperpriors on mRatio mu_logMratio = Normal( "mu_logMratio", mu=0, tau=1, shape=(1), testval=np.random.rand() * 0.1 ) sigma_delta = HalfNormal("sigma_delta", tau=1, shape=(1)) delta_tilde = Normal("delta_tilde", mu=0, sigma=1, shape=(nSubj, 1)) delta = Deterministic("delta", sigma_delta * delta_tilde) epsilon_logMratio = Beta("epsilon_logMratio", 1, 1, shape=(1)) logMratio = Deterministic("logMratio", mu_logMratio + epsilon_logMratio * delta) mRatio = Deterministic("mRatio", math.exp(logMratio)) # Type 2 priors meta_d = Deterministic("meta_d", mRatio * d1) # Specify ordered prior on criteria # bounded above and below by Type 1 c1 cS1_hn = Normal( "cS1_hn", mu=0, sigma=1, shape=(nSubj, nRatings - 1), testval=np.linspace(-1.5, -0.5, nRatings - 1) .reshape(1, nRatings - 1) .repeat(nSubj, axis=0), ) cS1 = Deterministic("cS1", -mu_c2 + (cS1_hn * sigma_c2)) cS2_hn = Normal( "cS2_hn", mu=0, sigma=1, shape=(nSubj, nRatings - 1), testval=np.linspace(0.5, 1.5, nRatings - 1) .reshape(1, nRatings - 1) .repeat(nSubj, axis=0), ) cS2 = Deterministic("cS2", mu_c2 + (cS2_hn * sigma_c2)) # Means of SDT distributions S2mu = meta_d / 2 S1mu = -meta_d / 2 # Calculate normalisation constants C_area_rS1 = cumulative_normal(c1 - S1mu) I_area_rS1 = cumulative_normal(c1 - S2mu) C_area_rS2 = 1 - cumulative_normal(c1 - S2mu) I_area_rS2 = 1 - cumulative_normal(c1 - S1mu) # Get nC_rS1 probs nC_rS1 = cumulative_normal(cS1 - S1mu) / C_area_rS1 nC_rS1 = Deterministic( "nC_rS1", math.concatenate( ( [ cumulative_normal(cS1[:, 0].reshape((nSubj, 1)) - S1mu) / C_area_rS1, nC_rS1[:, 1:] - nC_rS1[:, :-1], ( ( cumulative_normal(c1 - S1mu) - cumulative_normal( cS1[:, nRatings - 2].reshape((nSubj, 1)) - S1mu ) ) / C_area_rS1 ), ] ), axis=1, ), ) # Get nI_rS2 probs nI_rS2 = (1 - cumulative_normal(cS2 - S1mu)) / I_area_rS2 nI_rS2 = Deterministic( "nI_rS2", math.concatenate( ( [ ( (1 - cumulative_normal(c1 - S1mu)) - ( 1 - cumulative_normal( cS2[:, 0].reshape((nSubj, 1)) - S1mu ) ) ) / I_area_rS2, nI_rS2[:, :-1] - (1 - cumulative_normal(cS2[:, 1:] - S1mu)) / I_area_rS2, ( 1 - cumulative_normal( cS2[:, nRatings - 2].reshape((nSubj, 1)) - S1mu ) ) / I_area_rS2, ] ), axis=1, ), ) # Get nI_rS1 probs nI_rS1 = (-cumulative_normal(cS1 - S2mu)) / I_area_rS1 nI_rS1 = Deterministic( "nI_rS1", math.concatenate( ( [ cumulative_normal(cS1[:, 0].reshape((nSubj, 1)) - S2mu) / I_area_rS1, nI_rS1[:, :-1] + (cumulative_normal(cS1[:, 1:] - S2mu)) / I_area_rS1, ( cumulative_normal(c1 - S2mu) - cumulative_normal( cS1[:, nRatings - 2].reshape((nSubj, 1)) - S2mu ) ) / I_area_rS1, ] ), axis=1, ), ) # Get nC_rS2 probs nC_rS2 = (1 - cumulative_normal(cS2 - S2mu)) / C_area_rS2 nC_rS2 = Deterministic( "nC_rS2", math.concatenate( ( [ ( (1 - cumulative_normal(c1 - S2mu)) - ( 1 - cumulative_normal( cS2[:, 0].reshape((nSubj, 1)) - S2mu ) ) ) / C_area_rS2, nC_rS2[:, :-1] - ((1 - cumulative_normal(cS2[:, 1:] - S2mu)) / C_area_rS2), ( 1 - cumulative_normal( cS2[:, nRatings - 2].reshape((nSubj, 1)) - S2mu ) ) / C_area_rS2, ] ), axis=1, ), ) # Avoid underflow of probabilities nC_rS1 = math.switch(nC_rS1 < Tol, Tol, nC_rS1) nI_rS2 = math.switch(nI_rS2 < Tol, Tol, nI_rS2) nI_rS1 = math.switch(nI_rS1 < Tol, Tol, nI_rS1) nC_rS2 = math.switch(nC_rS2 < Tol, Tol, nC_rS2) # TYPE 2 SDT MODEL (META-D) # Multinomial likelihood for response counts ordered as c(nR_S1,nR_S2) Multinomial( "CR_counts", cr, nC_rS1, shape=(nSubj, nRatings), observed=counts[:, :nRatings], ) Multinomial( "FA_counts", FA, nI_rS2, shape=(nSubj, nRatings), observed=counts[:, nRatings : nRatings * 2], ) Multinomial( "M_counts", m, nI_rS1, shape=(nSubj, nRatings), observed=counts[:, nRatings * 2 : nRatings * 3], ) Multinomial( "H_counts", H, nC_rS2, shape=(nSubj, nRatings), observed=counts[:, nRatings * 3 : nRatings * 4], ) if sample_model is True: trace = sample(return_inferencedata=True, **kwargs) return model, trace else: return model
def get_mixbml_model(xs, hparams, verbose=0): u"""Returns a PyMC3 probabilistic model of mixture BML. This function should be invoked within a Model session of PyMC3. :param xs: Observation data. :type xs: ndarray, shape=(n_samples, 2), dtype=float :param hparams: Hyperparameters for inference. :type hparams: MixBMLParams :return: Probabilistic model :rtype: pymc3.Model """ # Standardize samples floatX = 'float32' # TODO: remove literal n_samples = xs.shape[0] xs = xs.astype(floatX) xs = standardize_samples(xs, True) if hparams.standardize else xs # Common scaling parameters std_x = np.std(xs, axis=0).astype(floatX) max_c = 1.0 # TODO: remove literal tau_cmmn = np.array([(std_x[0] * max_c)**2, (std_x[1] * max_c)**2]).astype(floatX) # Prior of individual specific effects (\tilde{\mu}_{l}^{(i)}) L_cov = _get_L_cov(hparams) mu1s, mu2s = _indvdl_t(hparams, std_x, n_samples, L_cov) # Noise variance h1, h2 = _noise_variance(hparams, tau_cmmn) # Common interceptions mu1, mu2 = _common_interceptions(hparams, tau_cmmn) # Noise model # obs1 (obs2) is a log likelihood function, not RV obs1, obs2 = _noise_model(hparams, h1, h2) # Pair of causal models v1_params = [mu1, mu1s, obs1] v2_params = [mu2, mu2s, obs2] # lp_m1: x1 -> x2 (b_21 is non-zero) # lp_m2: x2 -> x1 (b_12 is non-zero) lp_m1 = _causal_model(hparams, v1_params, v2_params, tau_cmmn, '21') lp_m2 = _causal_model(hparams, v2_params, v1_params, tau_cmmn, '12') # Prior of mixing proportions for causal models z = Beta('z', alpha=1, beta=1) # Mixture of potentials of causal models def lp_mix(xs): def flip(xs): # Filp 1st and 2nd features return tt.stack([xs[:, 1], xs[:, 0]], axis=0).T return pm.logsumexp( tt.stack([tt.log(z) + lp_m1(xs), tt.log(1 - z) + lp_m2(flip(xs))], axis=0)) DensityDist('dist', lp_mix, observed=xs)
Q = DiscreteObsMJP_unif_prior('Q', M=M, lower=0.0, upper=1.0, shape=(M, M)) #S = DiscreteObsMJP('S', pi=pi, Q=Q, M=M, nObs=nObs, observed_jumps=obs_jumps, T=T, shape=(nObs), testval=np.ones(nObs,dtype='int32')) S = DiscreteObsMJP('S', pi=pi, Q=Q, M=M, nObs=nObs, observed_jumps=obs_jumps, T=T, shape=(nObs)) #B0 = Beta('B0', alpha = 1., beta = 1., shape=(K,M), testval=0.2*np.ones((K,M))) #B = Beta('B', alpha = 1., beta = 1., shape=(K,M), testval=0.2*np.ones((K,M))) B0 = Beta('B0', alpha=1., beta=1., shape=(K, M)) B = Beta('B', alpha=1., beta=1., shape=(K, M)) #X = Comorbidities('X', S=S, B0=B0,B=B, T=T, shape=(nObs, K), testval=np.ones((nObs,K),dtype='int8')) X = Comorbidities('X', S=S, B0=B0, B=B, T=T, shape=(nObs, K)) #Z = Beta('Z', alpha = 0.1, beta = 1., shape=(K,D), testval=0.5*np.ones((K,D))) #L = Beta('L', alpha = 1., beta = 1., shape=D, testval=0.5*np.ones(D)) Z = Beta('Z', alpha=0.1, beta=1., shape=(K, D)) L = Beta('L', alpha=1., beta=1., shape=D) O_obs = Claims('O_obs', X=X, Z=Z, L=L, T=T, D=D,
if FAKE_DATA: faker = gen.Test_Generator() data = faker.gen('soft1') else: data = read.get_all_sets() pathways, features, path_dict, reverse_path_dict, evidence, metfrag_evidence = data print("num_pathways:", len(pathways)) print("num_features:", len(features)) print("num_evidence:", len(evidence)) print("num_metfrag: ", len(metfrag_evidence)) rate_prior = 0.5 model = Model() with model: eps = Beta('eps', 0.005, 1) ap = {p: Gamma('p_' + p, rate_prior, 1) for p in pathways} bmp = { p: { feat: Gamma('b_{' + p + ',' + feat + '}', ap[p], 1) for feat in path_dict[p] } for p in pathways } y_bmp = {} g = {} def logp_f(f, b, eps): if f in evidence: return math.log(1 - math.e**(-1 * b) + epsilon) if f in metfrag_evidence:
def setUp(self): #test Claims N = 100 # Number of patients M = 6 # Number of hidden states K = 10 # Number of comorbidities D = 721 # Number of claims Dd = 80 # Maximum number of claims that can occur at once min_obs = 10 # Minimum number of observed claims per patient max_obs = 30 # Maximum number of observed claims per patient self.M = M self.N = N self.K = K # Load pre-generated data from pickle import load T = load(open('../../data/X_layer_100_patients_old/T.pkl', 'rb')) self.T = T obs_jumps = load( open('../../data/X_layer_100_patients_old/obs_jumps.pkl', 'rb')) S_start = load(open('../../data/X_layer_100_patients_old/S.pkl', 'rb')) X_start = load(open('../../data/X_layer_100_patients_old/X.pkl', 'rb')) Z_start = load(open('../../data/X_layer_100_patients_old/Z.pkl', 'rb')) L_start = load(open('../../data/X_layer_100_patients_old/L.pkl', 'rb')) O = load(open('../../data/X_layer_100_patients_old/O_input.pkl', 'rb')) self.nObs = nObs = T.sum() self.zeroIndices = np.roll(self.T.cumsum(), 1) self.zeroIndices[0] = 0 obs_jumps = np.hstack([np.zeros((N, 1), dtype='int8'), obs_jumps]) obs_jumps = np.concatenate([obs_jumps[i, 0:T[i]] for i in range(N)]) O = np.concatenate([O[:, 0:T[i], i].T for i in range(N)]) S_start = np.concatenate([S_start[i, 0:T[i]] for i in range(N)]) X_start = np.concatenate([X_start[:, 0:T[i], i].T for i in range(N)]) anchors = [] self.Z_original mask = np.ones((K, D)) for anchor in anchors: for hold in anchor[1]: mask[:, hold] = 0 mask[anchor[0], hold] = 1 Z_start = Z_start[mask.nonzero()] with Model() as self.model: self.pi = Dirichlet('pi', a=as_tensor_variable( [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]), shape=M) pi_min_potential = Potential( 'pi_min_potential', TT.switch(TT.min(self.pi) < .1, -np.inf, 0)) self.Q = DiscreteObsMJP_unif_prior('Q', M=M, lower=0.0, upper=1.0, shape=(M, M)) self.S = DiscreteObsMJP('S', pi=self.pi, Q=self.Q, M=M, nObs=nObs, observed_jumps=obs_jumps, T=T, shape=(nObs)) self.B0 = Beta('B0', alpha=1., beta=1., shape=(K, M)) self.B = Beta('B', alpha=1., beta=1., shape=(K, M)) self.X = Comorbidities('X', S=self.S, B0=self.B0, B=self.B, T=T, shape=(nObs, K)) #self.Z = Beta('Z', alpha = 0.1, beta = 1., shape=(K,D)) self.Z = Beta_with_anchors('Z', anchors=anchors, K=K, D=D, alpha=0.1, beta=1., shape=(K, D)) self.L = Beta('L', alpha=1., beta=1., shape=D) self.testClaims = Claims('O_obs', X=self.X, Z=self.Z, L=self.L, T=T, D=D, O_input=O, shape=(nObs, Dd), observed=O) self.forS = ForwardS(vars=[self.S], N=N, T=T, nObs=nObs, observed_jumps=obs_jumps) self.forX = ForwardX(vars=[self.X], N=N, T=T, K=K, D=D, Dd=Dd, O=O, nObs=nObs) from scipy.special import logit self.Q_raw_log = logit( np.array([0.631921, 0.229485, 0.450538, 0.206042, 0.609582])) B_lo = logit( np.array( [[0.000001, 0.760000, 0.720000, 0.570000, 0.700000, 0.610000], [0.000001, 0.460000, 0.390000, 0.220000, 0.200000, 0.140000], [0.000001, 0.620000, 0.620000, 0.440000, 0.390000, 0.240000], [0.000001, 0.270000, 0.210000, 0.170000, 0.190000, 0.070000], [0.000001, 0.490000, 0.340000, 0.220000, 0.160000, 0.090000], [0.000001, 0.620000, 0.340000, 0.320000, 0.240000, 0.120000], [0.000001, 0.550000, 0.390000, 0.320000, 0.290000, 0.150000], [0.000001, 0.420000, 0.240000, 0.170000, 0.170000, 0.110000], [0.000001, 0.310000, 0.300000, 0.230000, 0.190000, 0.110000], [0.000001, 0.470000, 0.340000, 0.190000, 0.190000, 0.110000]])) B0_lo = logit( np.array( [[0.410412, 0.410412, 0.418293, 0.418293, 0.429890, 0.429890], [0.240983, 0.240983, 0.240983, 0.240983, 0.240983, 0.240983], [0.339714, 0.339714, 0.339714, 0.339714, 0.339714, 0.339714], [0.130415, 0.130415, 0.130415, 0.130415, 0.130415, 0.130415], [0.143260, 0.143260, 0.143260, 0.143260, 0.143260, 0.143260], [0.211465, 0.211465, 0.211465, 0.211465, 0.211465, 0.211465], [0.194187, 0.194187, 0.194187, 0.194187, 0.194187, 0.194187], [0.185422, 0.185422, 0.185422, 0.185422, 0.185422, 0.185422], [0.171973, 0.171973, 0.171973, 0.171973, 0.171973, 0.171973], [0.152277, 0.152277, 0.152277, 0.152277, 0.152277, 0.152277]])) Z_lo = logit(Z_start) L_lo = logit(L_start) #import pdb; pdb.set_trace() self.myTestPoint = { 'Q_ratematrixoneway': self.Q_raw_log, 'B_logodds': B_lo, 'B0_logodds': B0_lo, 'S': S_start, 'X': X_start, 'Z_anchoredbeta': Z_lo, 'L_logodds': L_lo, 'pi_stickbreaking': np.array([0.5, 0.5, 0.5, 0.5, 0.5, 0.5]) }
# exp(t*Q)[m,m']: probability of transitioning from disease state m to m' after a period of time t Q = DiscreteObsMJP_unif_prior('Q', M=M, lower=0.0, upper=1.0, shape=(M, M)) # S[o]: disease state (between 0 and M-1) at obeservation o # S = DiscreteObsMJP('S', pi=pi, Q=Q, M=M, nObs=nObs, observed_jumps=obs_jumps, T=T, shape=(nObs)) B0 = Beta('B0', alpha=1., beta=1., shape=(K, M)) B0_monotonicity_constraint = Potential( 'B0_monotonicity_constraint', TT.switch(TT.min(DES_diff(B0)) < 0., 100.0 * TT.min(DES_diff(B0)), 0)) B = Beta('B', alpha=1., beta=1., shape=(K, M)) X = Comorbidities('X', S=S, B0=B0, B=B, T=T, shape=(nObs, K)) #Z = Beta('Z', alpha = 0.1, beta = 1., shape=(K,D)) Z = Beta_with_anchors('Z', anchors=anchors, K=K, D=D, alpha=0.1, beta=1.,
X = np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], dtype=np.int8) model = Model() Q_test = np.array([[-6, 2, 2, 1, 1], [1, -4, 0, 1, 2], [1, 0, -4, 2, 1], [2, 1, 0, -3, 0], [1, 1, 1, 1, -4]]) with model: Q = DiscreteObsMJP_unif_prior('Q', M=M, shape=(M, M)) S = DiscreteObsMJP('S', Q=Q, observed_jumps=observed_jumps, shape=(Tn)) B0 = Beta('B0', alpha=1, beta=1, shape=(K, M)) B = Beta('B', alpha=1, beta=1, shape=(K, M)) Xobs = Comorbidities('Xobs', S=S, B0=B0, B=B, shape=(K, Tn), observed=X) step2 = ForwardS(vars=[S], X=X, observed_jumps=observed_jumps) step2.step_sizes = np.array([0.1, 1, 100]) pS = step2.compute_pS(Q_test, 5) print pS
def setUp(self): #test Claims N = 5 # Number of patients self.N = N M = 3 # Number of hidden states self.M = M K = 2 # Number of comorbidities D = 20 # Number of claims Dd = 4 # Maximum number of claims that can occur at once min_obs = 2 # Minimum number of observed claims per patient max_obs = 4 # Maximum number of observed claims per patient #obs_jumps = np.ones((N,max_obs-1)) obs_jumps = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]]) T = np.array([4, 2, 3, 4, 2]) self.T = T nObs = T.sum() obs_jumps = np.hstack([np.zeros((N, 1), dtype='int8'), obs_jumps]) obs_jumps = np.concatenate([obs_jumps[i, 0:T[i]] for i in range(N)]) #O(4,4,5) #O = np.zeros((nObs,Dd),dtype='int8') O = np.zeros((Dd, max_obs, N), dtype='int8') #import pdb; pdb.set_trace() O[[0, 1, 3, 2, 3, 3], [0, 1, 3, 2, 3, 3], [0, 1, 4, 3, 3, 4]] = 1 #O[[0,5,11,12],[0,1,2,3]] = 1 O = np.concatenate([O[:, 0:T[i], i].T for i in range(N)]) Z_lo = np.array([[ -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509 ], [ -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509, -2.30258509 ]]) anchors = [] mask = np.ones((K, D)) for anchor in anchors: for hold in anchor[1]: mask[:, hold] = 0 mask[anchor[0], hold] = 1 Z_lo = Z_lo[mask.nonzero()] with Model() as self.model: self.pi = Dirichlet('pi', a=as_tensor_variable([0.5, 0.5, 0.5]), shape=M) pi_min_potential = Potential( 'pi_min_potential', TT.switch(TT.min(self.pi) < .1, -np.inf, 0)) self.Q = DiscreteObsMJP_unif_prior('Q', M=M, lower=0.0, upper=1.0, shape=(M, M)) self.S = DiscreteObsMJP('S', pi=self.pi, Q=self.Q, M=M, nObs=nObs, observed_jumps=obs_jumps, T=T, shape=(nObs)) self.B0 = Beta('B0', alpha=1., beta=1., shape=(K, M)) self.B = Beta('B', alpha=1., beta=1., shape=(K, M)) self.X = Comorbidities('X', S=self.S, B0=self.B0, B=self.B, T=T, shape=(nObs, K)) #self.Z = Beta('Z', alpha = 0.1, beta = 1., shape=(K,D)) self.Z = Beta_with_anchors('Z', anchors=anchors, K=K, D=D, alpha=0.1, beta=1., shape=(K, D)) self.L = Beta('L', alpha=1., beta=1., shape=D) #L = Beta('L', alpha = 0.1, beta = 1, shape=D, transform=None) #L = Uniform('L', left = 0.0, right = 1.0, shape=D, transform=None) #L = Uniform('L', lower = 0.0, upper = 1.0, shape=D) self.testClaims = Claims('O_obs', X=self.X, Z=self.Z, L=self.L, T=T, D=D, O_input=O, shape=(nObs, Dd), observed=O) self.forS = ForwardS(vars=[self.S], N=N, T=T, nObs=nObs, observed_jumps=obs_jumps) self.forX = ForwardX(vars=[self.X], N=N, T=T, K=K, D=D, Dd=Dd, O=O, nObs=nObs) self.myTestPoint = { 'Z_anchoredbeta': Z_lo, 'Q_ratematrixoneway': np.array([0.1, 0.1]), 'pi_stickbreaking': np.array([0.2, 0.1]), 'S': np.array([[0, 0, 1, 1], [1, 1, 1, 1], [1, 1, 2, 2], [0, 2, 2, 2], [0, 0, 0, 1]], dtype=np.int32), 'B0_logodds': np.array([[0., 1., 0.], [0., 0., 1.]]), 'X': np.array([[[0, 1, 1, 1, 1], [0, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]], [[1, 1, 0, 0, 1], [1, 1, 0, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]]], dtype=np.int8), 'L_logodds': np.array([ 0.1, 0.1, 0.1, 0.1, 0.01, 0.01, 0.01, 0.01, 0.0011, 0.0011, 0.0011, 0.0011, 0.0011, 0., 0.0101, 0.0101, 0.0101, 0.01, 0.01, 0.01 ]), 'B_logodds': np.array([[1., 0., 1.], [0., 1., 0.]]) } self.myTestPoint['S'] = np.concatenate( [self.myTestPoint['S'][i, 0:T[i]] for i in range(N)]) self.myTestPoint['X'] = np.concatenate( [self.myTestPoint['X'][:, 0:T[i], i].T for i in range(N)]) stepX_Correct = np.array([[[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1]], [[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]], dtype=np.int8) stepX_Correct = np.array([[[0, 0, 0, 0, 0], [0, 0, 0, 1, 0], [0, 0, 0, 1, 0], [0, 0, 0, 1, 0]], [[0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 0], [0, 1, 0, 0, 1]]], dtype=np.int8)
def run_phi(data, **kwargs): if isinstance(data, str): data = csv(data) data = np.array(data) # Check limits in **kwargs if kwargs.get("limits") is not None: limits = kwargs.get("limits") else: limits = (np.nanmin(list(itertools.chain.from_iterable(data))), np.nanmax(list(itertools.chain.from_iterable(data)))) if kwargs.get("verbose") is not None: verbose = kwargs.get("verbose") else: verbose = False if (kwargs.get("binning") is not None) and not kwargs.get("binning"): print("removing binning on borders") binning_multiplier = 2 else: binning_multiplier = 1 if kwargs.get("seed") is not None: seed = kwargs.get("seed") else: seed = 123 if kwargs.get("table") is not None: table = kwargs.get("table") else: table = False if kwargs.get("N") is not None: N = kwargs.get("N") else: N = 1000 if kwargs.get("keep_missing") is not None: keep_missing = kwargs.get("keep_missing") else: keep_missing = None #AUTO #keep_missing = True if kwargs.get("fast") is not None: fast = kwargs.get("fast") else: fast = True if kwargs.get("njobs") is not None: njobs = kwargs.get("njobs") else: njobs = 2 if kwargs.get("sd") is not None: sd = kwargs.get("sd") else: sd = 1000000 # Check gt in **kwargs if kwargs.get("gt") is not None: gt = kwargs.get("gt") else: gt = [None] * len(data) if verbose: print("Computing Phi") idx_of_gt = np.array([x is not None for x in gt]) idx_of_not_gt = np.array([x is None for x in gt]) num_of_gt = np.sum(idx_of_gt) basic_model = Model() for i, g in enumerate(gt): if g is not None: gt[i] = scale_mat(np.array([[gt[i]] * len(data[i])]), limits, binning_multiplier=binning_multiplier)[0][0] num_of_docs = len(data) # number of documents rectangular = True sparse = False if np.isnan(data).any(): sparse = True data = np.ma.masked_invalid(data) data = minimal_matrix(data) scaled = scale_mat(data, limits, binning_multiplier=binning_multiplier) if (np.count_nonzero(np.isnan(scaled)) / scaled.size) > 0.2: # a lot of nans if verbose: print( "WARNING: a lot of missing values: we are going to set keep_missing=False to improve convergence (if not manually overridden)" ) if keep_missing is None: keep_missing = False if (sparse and keep_missing == False): rectangular = False scaled = [doc[~np.isnan(doc)].tolist() for doc in scaled] #make data a list of lists NUM_OF_ITERATIONS = N with basic_model: precision = Normal('precision', mu=2, sd=sd) #precision = Gamma('precision',mu=2,sd=1) if num_of_docs - num_of_gt == 1: mu = Normal('mu', mu=1 / 2, sd=sd) else: mu = Normal('mu', mu=1 / 2, sd=sd, shape=num_of_docs - num_of_gt) alpha = mu * precision beta = precision * (1 - mu) if rectangular: masked = pd.DataFrame( scaled[idx_of_not_gt]) #needed to keep nan working if num_of_docs - num_of_gt == 1: Beta('beta_obs', observed=masked, alpha=alpha, beta=beta) else: Beta('beta_obs', observed=masked.T, alpha=alpha, beta=beta, shape=num_of_docs - num_of_gt) else: for i, doc in enumerate(scaled): Beta('beta_obs' + str(i), observed=doc, alpha=alpha[i], beta=beta[i]) for i, g in enumerate(gt): if g is not None: mu = Normal('mu' + str(i), mu=gt[i], sd=1) alpha = mu * precision beta = precision * (1 - mu) Beta('beta_obs_g' + str(i), observed=scaled[i], alpha=alpha, beta=beta) #alpha=a,beta=b,observed=beta) try: if fast: assert False stds = np.ones(basic_model.ndim) for _ in range(5): args = {'scaling': stds**2, 'is_cov': True} trace = pm.sample(round(NUM_OF_ITERATIONS / 10), tune=round(NUM_OF_ITERATIONS / 10), init=None, nuts_kwargs=args, chains=10, progressbar=verbose, random_seed=seed) samples = [basic_model.dict_to_array(p) for p in trace] stds = np.array(samples).std(axis=0) step = pm.NUTS(scaling=stds**2, is_cov=True, target_accept=0.9) start = trace[0] trace = sample(NUM_OF_ITERATIONS, tune=round(NUM_OF_ITERATIONS / 2), njobs=njobs, chains=8, init=None, step=step, start=start, progressbar=verbose, random_seed=seed) # Staistical inference beg = time() #start = find_MAP() bef_slice = time() #step = NUTS()# Metropolis() #step = Metropolis() aft_slice = time() bef_trace = time() #trace = sample(NUM_OF_ITERATIONS, progressbar=verbose,random_seed=123, njobs=njobs,start=start,step=step) # trace = sample(NUM_OF_ITERATIONS, progressbar=verbose,random_seed=123, njobs=njobs,init=None,tune=100) except: beg = time() step = Metropolis() #start = find_MAP() trace = sample(NUM_OF_ITERATIONS, progressbar=verbose, random_seed=seed, njobs=njobs, step=step) #,start=start) #pm.summary(trace,include_transformed=True) if np.float(pymc3.__version__) <= 3.3: res = pm.stats.df_summary(trace, include_transformed=True) else: res = pm.summary(trace, include_transformed=True) res.drop(["sd", "mc_error"], axis=1, inplace=True) res = res.transpose() res["agreement"] = agreement(res['precision']) # ---- #sub_res = res.copy() # Mu rescaling col_agreement = res["agreement"] col_precision = res["precision"] res.drop("agreement", inplace=True, axis=1) res.drop("precision", inplace=True, axis=1) if table: col_names = res.columns[0:len(data) - 1] for i, name in enumerate(col_names): l = len(scaled[i]) * binning_multiplier for j in range(3): b = res[name].iloc[j] mu_res = (b * l - 0.5) / (l - 1) res[name].iloc[j] = np.clip(mu_res, 0, 1) * (limits[1] - limits[0]) res["agreement"] = col_agreement res.insert(0, "precision", col_precision) aft_trace = time() computation_time = time() - beg if verbose: print("Elapsed time for computation: ", computation_time) convergence = True if np.isnan(res.loc['Rhat']['precision'] ) or np.abs(res.loc['Rhat']['precision'] - 1) > 1e-1: print("Warning! You need more iterations!") convergence = False if table: return { 'agreement': col_agreement['mean'], 'interval': col_agreement[['hpd_2.5', 'hpd_97.5']].values, "computation_time": computation_time, "convergence_test": convergence, 'table': res } else: return { 'agreement': col_agreement['mean'], 'interval': col_agreement[['hpd_2.5', 'hpd_97.5']].values, "computation_time": computation_time, "convergence_test": convergence }