def plot(f, x_start, x_end, x_interval=None, epsilon=0.15): if x_interval is None: x_interval = x_end - x_start x = py.arange(x_start, x_end + 0.1 * x_interval, x_interval) # y = x^2 py.plot(x, f(x), 'ko', label='$y=x^2-10$') # y = 0 py.plot(x, py.zeros_like(x), 'ro', label='$y=0$') # +/- epsilon py.plot(x, epsilon * py.ones_like(x), 'r-.', label=r'$+\epsilon$') py.plot(x, -epsilon * py.ones_like(x), 'r--', label=r'$-\epsilon$') # x 축 이름표 # x axis label py.xlabel('x') # y 축 이름표 # y axis label py.ylabel('y') # 범례 표시 # Show legend py.legend() # 모눈 표시 # Indicate grid py.grid() return x
def alphas_hist(df, outdir=FIGS_DIR, categories=None): subset_cols = ['neuron name', 'neuron type'] if categories != None: for category in categories: if category != 'neuron type': subset_cols.append(category) df2 = df.drop_duplicates(subset_cols) alphas = None weights = None labels = None if categories == None: alphas = list(df2['alpha']) print "all neurons mean alpha", pylab.mean(alphas), "+/-", pylab.std( alphas, ddof=1) weights = pylab.ones_like(alphas) / len(alphas) else: alphas = [] weights = [] labels = [] for name, group in df2.groupby(categories): cat_alphas = group['alpha'] print name + " neurons mean alpha", pylab.mean(cat_alphas) cat_weights = pylab.ones_like(cat_alphas) / len(cat_alphas) alphas.append(cat_alphas) weights.append(cat_weights) labels.append(name) pylab.figure() sns.set() if labels == None: pylab.hist(alphas, range=(0, 1), weights=weights) else: pylab.hist(alphas, range=(0, 1), weights=weights, label=labels) pylab.legend() curr_ax = pylab.gca() curr_ax.set_ylim((0, 1)) pylab.xlabel('alpha', size=20) pylab.ylabel('proportion', size=20) pylab.xticks(fontsize=20) pylab.yticks(fontsize=20) pylab.tight_layout() name = 'alphas_hist' if categories != None: cat_str = '_'.join(categories) cat_str = cat_str.replace(' ', '_') name += '_' + cat_str outname = '%s/%s.pdf' % (outdir, name) outname = outname.replace(' ', '_') pylab.savefig('%s/%s.pdf' % (outdir, name), format='pdf') pylab.close()
def panel(): global Ras, Rms, Ie, Ies global tstart global pan, t0, t1 #Pulse start time tstart = 0.1 #Ras = [1.,500.,20.] #Ohm*cm #Rms = [200.,10000.,200.] #Ohm*cm^2 Ras = [5.0, 500.0, 50.0] #Ohm*cm Rms = [200.0, 10000.0, 1000.0] #Ohm*cm^2 Ie = 4. #nA Ies = pl.c_[pl.ones_like(ns.t)] Ies[:pl.ceil(tstart / ns.dt)] = 0. t0 = 0.101 t1 = 0.6 pan = simulationpanel.SimulationPanel() pan.Move((640, 600)) pan.setdict(globals()) pan.addcommand('sim()') pan.addcommand('plottao()') pan.addvar('Ras') pan.addvar('Rms') pan.addvar('Ie') pan.addvar('t0') pan.addvar('t1')
def fit_age_standardizing_model(model): """Age-standardizing model""" ## Spline model to represent age-specific rate model.vars += dismod3.age_pattern.spline( name="std", ages=model.ages, knots=knots, smoothing=pl.inf, interpolation_method="linear" ) model.vars += dismod3.age_group.age_standardize_approx( name="std", ages=model.ages, age_weights=pl.ones_like(model.ages), mu_age=model.vars["mu_age"], age_start=model.input_data["age_start"], age_end=model.input_data["age_end"], ) model.vars += {"delta": mc.Uniform("delta_std", 0.0, 1000.0, value=10.0)} ## Negative binomial rate model model.vars += dismod3.rate_model.neg_binom( name="std", pi=model.vars["mu_interval"], delta=model.vars["delta"], p=model.input_data["value"], n=model.input_data["effective_sample_size"], ) fit_model(model)
def plot_funnel(pi_true, delta_str): delta = float(delta_str) n = pl.exp(mc.rnormal(10, 2**-2, size=10000)) p = pi_true*pl.ones_like(n) # old way: #delta = delta * p * n nb = rate_model.neg_binom_model('funnel', pi_true, delta, p, n) r = nb['p_pred'].value pl.vlines([pi_true], .1*n.min(), 10*n.max(), linewidth=5, linestyle='--', color='black', zorder=10) pl.plot(r, n, 'o', color=colors[0], ms=10, mew=0, alpha=.25) pl.semilogy(schiz['r'], schiz['n'], 's', mew=1, mec='white', ms=15, color=colors[1], label='Observed Values') pl.xlabel('Rate (Per 1000 PY)', size=32) pl.ylabel('Study Size (PY)', size=32) pl.axis([-.0001, .0101, 50., 15000000]) pl.title(r'$\delta = %s$'%delta_str, size=48) pl.xticks([0, .005, .01], [0, 5, 10], size=30) pl.yticks(size=30)
def plot_funnel(pi_true, delta_str): delta = float(delta_str) n = pl.exp(mc.rnormal(10, 2**-2, size=10000)) p = pi_true * pl.ones_like(n) # old way: #delta = delta * p * n nb = rate_model.neg_binom_model('funnel', pi_true, delta, p, n) r = nb['p_pred'].value pl.vlines([pi_true], .1 * n.min(), 10 * n.max(), linewidth=5, linestyle='--', color='black', zorder=10) pl.plot(r, n, 'o', color=colors[0], ms=10, mew=0, alpha=.25) pl.semilogy(schiz['r'], schiz['n'], 's', mew=1, mec='white', ms=15, color=colors[1], label='Observed Values') pl.xlabel('Rate (Per 1000 PY)', size=32) pl.ylabel('Study Size (PY)', size=32) pl.axis([-.0001, .0101, 50., 15000000]) pl.title(r'$\delta = %s$' % delta_str, size=48) pl.xticks([0, .005, .01], [0, 5, 10], size=30) pl.yticks(size=30)
def fit_age_standardizing_model(model): """Age-standardizing model""" ## Spline model to represent age-specific rate model.vars += dismod3.age_pattern.spline(name='std', ages=model.ages, knots=knots, smoothing=pl.inf, interpolation_method='linear') model.vars += dismod3.age_group.age_standardize_approx(name='std', ages=model.ages, age_weights=pl.ones_like(model.ages), mu_age=model.vars['mu_age'], age_start=model.input_data['age_start'], age_end=model.input_data['age_end']) delta = mc.Uniform('delta_mid', 0., 1000., value=10.) model.vars += {'delta': delta} ## Negative binomial rate model model.vars += dismod3.rate_model.neg_binom(name='std', pi=model.vars['mu_interval'], delta=model.vars['delta'], p=model.input_data['value'], n=model.input_data['effective_sample_size']) fit_model(model)
def test_age_standardizing_approx(): # simulate data n = 50 sigma_true = .025 * pl.ones(n) a = pl.arange(0, 100, 1) pi_age_true = .0001 * (a * (100. - a) + 100.) ages = pl.arange(101) d = data_simulation.simulated_age_intervals('p', n, a, pi_age_true, sigma_true) # create model and priors vars = {} vars.update( age_pattern.age_pattern('test', ages, knots=pl.arange(0, 101, 5), smoothing=.01)) vars.update( age_integrating_model.age_standardize_approx( 'test', pl.ones_like(vars['mu_age'].value), vars['mu_age'], d['age_start'], d['age_end'], ages)) vars['pi'] = vars['mu_interval'] vars.update( rate_model.normal_model('test', pi=vars['pi'], sigma=0, p=d['value'], s=sigma_true)) # fit model m = mc.MCMC(vars) m.sample(3)
def panel(): global Ras,Rms,Ie,Ies global tstart global pan,t0,t1 #Pulse start time tstart = 0.1 #Ras = [1.,500.,20.] #Ohm*cm #Rms = [200.,10000.,200.] #Ohm*cm^2 Ras = [5.0, 500.0, 50.0] #Ohm*cm Rms = [200.0, 10000.0, 1000.0] #Ohm*cm^2 Ie = 4. #nA Ies = pl.c_[pl.ones_like(ns.t)] Ies[:pl.ceil(tstart/ns.dt)] = 0. t0 = 0.101 t1 = 0.6 pan = simulationpanel.SimulationPanel() pan.Move((640,600)) pan.setdict(globals()) pan.addcommand('sim()') pan.addcommand('plottao()') pan.addvar('Ras') pan.addvar('Rms') pan.addvar('Ie') pan.addvar('t0') pan.addvar('t1')
def SWE_Cgrid(beta, g, H, dx, yu, yv, dt, nt, u, v, h, h0): "solves the SWE using forward-backward time-stepping and the C-grid" "doubly periodic domain stating from u, v and h" [nx, ny] = py.shape(u) dy = yu[1] - yu[0] H = H * py.ones_like(h) # loop through all times for it in xrange(nt): # update old values uOld = u.copy() vOld = v.copy() hOld = h.copy() # alternate between calculating u or v first for iu in xrange(2): if iu % 2 == 0: # loop over x and y directions (using i and j) u = uOld + dt * (beta * yu * vatu(v) - g * ddxC(h + h0, dx)) else: v = vOld + dt * (-beta * yv * uatv(u) - g * ddyC(h + h0, dy)) # then calculate h using updated values of u and v h = hOld - dt * divC(hatu(H - h0) * u, hatv(H - h0) * v, dx, dy) return [u, v, h]
def test_covariate_model_sim_no_hierarchy(): # simulate normal data model = data.ModelData() model.hierarchy, model.output_template = data_simulation.small_output() X = mc.rnormal(0., 1.**2, size=(128, 3)) beta_true = [-.1, .1, .2] Y_true = pl.dot(X, beta_true) pi_true = pl.exp(Y_true) sigma_true = .01 * pl.ones_like(pi_true) p = mc.rnormal(pi_true, 1. / sigma_true**2.) model.input_data = pandas.DataFrame( dict(value=p, x_0=X[:, 0], x_1=X[:, 1], x_2=X[:, 2])) model.input_data['area'] = 'all' model.input_data['sex'] = 'total' model.input_data['year_start'] = 2000 model.input_data['year_end'] = 2000 # create model and priors vars = {} vars.update( covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model, 'all', 'total', 'all')) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2)
def test_covariate_model_sim_no_hierarchy(): # simulate normal data model = data.ModelData() model.hierarchy, model.output_template = data_simulation.small_output() X = mc.rnormal(0., 1.**2, size=(128,3)) beta_true = [-.1, .1, .2] Y_true = pl.dot(X, beta_true) pi_true = pl.exp(Y_true) sigma_true = .01*pl.ones_like(pi_true) p = mc.rnormal(pi_true, 1./sigma_true**2.) model.input_data = pandas.DataFrame(dict(value=p, x_0=X[:,0], x_1=X[:,1], x_2=X[:,2])) model.input_data['area'] = 'all' model.input_data['sex'] = 'total' model.input_data['year_start'] = 2000 model.input_data['year_end'] = 2000 # create model and priors vars = {} vars.update(covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model, 'all', 'total', 'all')) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2)
def fit_age_standardizing_model(model): """Age-standardizing model""" ## Spline model to represent age-specific rate model.vars += dismod3.age_pattern.spline(name='std', ages=model.ages, knots=knots, smoothing=pl.inf, interpolation_method='linear') model.vars += dismod3.age_group.age_standardize_approx( name='std', ages=model.ages, age_weights=pl.ones_like(model.ages), mu_age=model.vars['mu_age'], age_start=model.input_data['age_start'], age_end=model.input_data['age_end']) model.vars += {'delta': mc.Uniform('delta_std', 0., 1000., value=10.)} ## Negative binomial rate model model.vars += dismod3.rate_model.neg_binom( name='std', pi=model.vars['mu_interval'], delta=model.vars['delta'], p=model.input_data['value'], n=model.input_data['effective_sample_size']) fit_model(model)
def mortality(self, key): """ Calculate the all-cause mortality rate for the region and sex of disease_model, and return it in an array corresponding to age_mesh Parameters ---------- key : str, optional of the form 'all-cause_mortality+gbd_region+year+sex' data: list, optional the data list to extract all-cause mortality from """ data = self.filter_data(key.replace('world', 'all')) # TODO: generate world mortality rate and use it when key is world, instead of using replace-world-all hack if len(data) == 0: return NEARLY_ZERO * pl.ones_like(self.get_estimate_age_mesh()) else: ages = [] vals = [] for d in data: a0 = d.get('age_start', MISSING) y = self.value_per_1(d) if MISSING in [a0, y]: continue ages.append(a0) vals.append(y) # add one additional point so that interpolation is all within range of data ages.append(dismod3.settings.MAX_AGE) vals.append(y) m_all = dismod3.utils.interpolate(ages, vals, self.get_estimate_age_mesh()) return m_all
def alphas_hist(df): df2 = df.drop_duplicates(subset='name') alphas = list(df2['alpha']) weights = pylab.ones_like(alphas) / len(alphas) pylab.hist(alphas, range=(0, 1), weights=weights) curr_ax = pylab.gca() curr_ax.set_ylim((0, 1)) pylab.savefig('%s/alphas_hist.pdf' % OUTDIR, format='pdf')
def make_count_table(dm, wb): """Make a table representation of disease counts Parameters ---------- dm : DiseaseJson object wb : the xlwt Workbook to add the count table to """ ws = wb.add_sheet('Counts') params = 'incidence prevalence excess-mortality incidence_x_duration'.split() cols = 'region year sex age param mean lower upper'.split() for col, col_head in enumerate(cols): ws.write(0, col, col_head) row = 1 for region in dismod3.settings.gbd_regions: for year in dismod3.settings.gbd_years: for sex in dismod3.settings.gbd_sexes: for param in params: mean_array = dm.get_mcmc('mean', dismod3.utils.gbd_key_for(param, region, year, sex)) lower_array = dm.get_mcmc('lower_ui', dismod3.utils.gbd_key_for(param, region, year, sex)) upper_array = dm.get_mcmc('upper_ui', dismod3.utils.gbd_key_for(param, region, year, sex)) if len(mean_array) == 0: continue pop = population_by_region_year_sex(region, year, sex) if param == 'excess-mortality': # use estimate of prevalent population pop *= dm.get_mcmc('mean', dismod3.utils.gbd_key_for('prevalence', region, year, sex)) elif param == 'incidence_x_duration': pop = pl.ones_like(pop) # population already included in iX stoch for a in range(len(dismod3.settings.gbd_ages)-1): age = dismod3.settings.gbd_ages[a] ages = range(age, dismod3.settings.gbd_ages[a+1]) # special case for birth prevalence if param == 'prevalence': # increase end of age range for age==0 if age == 0: ages = [0,1] pop_weights = pop[ages] mean = dismod3.utils.rate_for_range(mean_array, ages, pop_weights) lower = dismod3.utils.rate_for_range(lower_array, ages, pop_weights) upper = dismod3.utils.rate_for_range(upper_array, ages, pop_weights) for col, val in enumerate(cols): ws.write(row, col, vars()[val]) row += 1
def chebspace1(a, b, n, returnWeights=False): """Chebyshev nodes for given degree n""" apb = 0.5 * (a + b) bma = 0.5 * (b - a) cs = apb - bma * cos(arange(1, 2*n, 2) * pi / (2*n)) if returnWeights: weights = ones_like(cs) weights = sin(arange(1, 2 * n, 2) * pi / (2 * n)) weights[1::2] = -1 * weights[1::2] return cs, weights else: return cs
def chebspace1(a, b, n, returnWeights=False): """Chebyshev nodes for given degree n""" apb = 0.5 * (a + b) bma = 0.5 * (b - a) cs = apb - bma * cos(arange(1, 2 * n, 2) * pi / (2 * n)) if returnWeights: weights = ones_like(cs) weights = sin(arange(1, 2 * n, 2) * pi / (2 * n)) weights[1::2] = -1 * weights[1::2] return cs, weights else: return cs
def plot_funnel(pi_true, delta_str): n = pl.exp(mc.rnormal(10, 2**-2, size=10000)) delta = float(delta_str) * pl.ones_like(n) p = pi_true * pl.ones_like(n) # old way: #delta = delta * p * n nb = rate_model.neg_binom('funnel', p, delta, p, n) r = nb['p_pred'].value pl.vlines([pi_true], .1 * n.min(), 10 * n.max(), linewidth=2, linestyle='-', color='w', zorder=9) pl.vlines([pi_true], .1 * n.min(), 10 * n.max(), linewidth=1, linestyle='--', color='black', zorder=10) pl.plot(r, n, 'ko', mew=0, alpha=.25) pl.semilogy(schiz['r'], schiz['n'], 'ks', mew=1, mec='white', ms=4, label='Observed values') pl.xlabel('Rate (per PY)') pl.ylabel('Study size (PY)') pl.xticks([0, .005, .01]) pl.axis([-.0001, .0101, 50., 15000000]) pl.title(r'$\delta = %s$' % delta_str)
def truncation_hist(df, outdir=FIGS_DIR): df2 = df[df['neuron type'].isin(['axon', 'truncated axon'])] df2 = df2.drop_duplicates(['neuron name', 'neuron type']) type_alphas = defaultdict(list) for neuron_name, group in df2.groupby('neuron name'): if len(group['neuron type']) < 2: continue for neuron_type, group2 in group.groupby('neuron type'): type_alphas[neuron_type] += list(group2['alpha']) alphas = [] weights = [] labels = [] for neuron_type in type_alphas: alpha = type_alphas[neuron_type] alphas.append(alpha) weights.append(pylab.ones_like(alpha) / float(len(alpha))) labels.append(neuron_type) pylab.figure() sns.set() pylab.hist(alphas, range=(0, 1), weights=weights, label=labels) leg = pylab.legend(frameon=True) pylab.setp(leg.get_texts(), fontsize=20) leg_frame = leg.get_frame() leg_frame.set_linewidth(5) leg_frame.set_edgecolor('k') curr_ax = pylab.gca() curr_ax.set_ylim((0, 1)) pylab.xlabel('alpha', size=30) pylab.ylabel('proportion', size=30) pylab.tight_layout() name = 'truncation_hist' outname = '%s/%s.pdf' % (outdir, name) outname = outname.replace(' ', '_') pylab.savefig('%s/%s.pdf' % (outdir, name), format='pdf') pylab.close() axons = pylab.array(type_alphas['axon']) truncated_axons = pylab.array(type_alphas['truncated axon']) differences = axons - truncated_axons print '-------------------------' print "Truncation test" print min(differences), pylab.median(differences), max(differences) print pylab.mean(differences), "+/-", pylab.std(differences, ddof=1) print wilcoxon(axons, truncated_axons) print ttest_rel(axons, truncated_axons)
def angles_stats(): df = pd.read_csv('%s/%s' % (OUTDIR, FNAME), skipinitialspace=True) df['degrees'] = degrees(df['angle']) deg = pylab.array(df['degrees']) weight = pylab.ones_like(deg) / float(len(deg)) pylab.figure() pylab.hist(deg, weights=weight) pylab.xlabel('degree') pylab.ylabel('proportion') pylab.savefig('branch_angles/branch_angles.pdf', format='pdf') pylab.close()
def plot_funnel(pi_true, delta_str): n = pl.exp(mc.rnormal(10, 2 ** -2, size=10000)) delta = float(delta_str) * pl.ones_like(n) p = pi_true * pl.ones_like(n) # old way: # delta = delta * p * n nb = rate_model.neg_binom("funnel", p, delta, p, n) r = nb["p_pred"].value pl.vlines([pi_true], 0.1 * n.min(), 10 * n.max(), linewidth=2, linestyle="-", color="w", zorder=9) pl.vlines([pi_true], 0.1 * n.min(), 10 * n.max(), linewidth=1, linestyle="--", color="black", zorder=10) pl.plot(r, n, "ko", mew=0, alpha=0.25) pl.semilogy(schiz["r"], schiz["n"], "ks", mew=1, mec="white", ms=4, label="Observed values") pl.xlabel("Rate (per PY)") pl.ylabel("Study size (PY)") pl.xticks([0, 0.005, 0.01]) pl.axis([-0.0001, 0.0101, 50.0, 15000000]) pl.title(r"$\delta = %s$" % delta_str)
def neuron_types_hist(df): df2 = df.drop_duplicates(subset='name') pylab.figure() for neuron_type, group in df2.groupby('neuron_type'): alphas = list(group['alpha']) weights = pylab.ones_like(alphas) / len(alphas) pylab.hist(alphas, alpha=0.5, label=neuron_type,\ range=(0, 1), weights=weights) pylab.legend() curr_ax = pylab.gca() curr_ax.set_ylim((0, 1)) pylab.savefig('%s/neuron_types_hist.pdf' % OUTDIR, format='pdf') pylab.close()
def runtimes_stats(): df = pd.read_csv('test_runtimes.csv', skipinitialspace=True) print "total trials" print len(df['algorithm']) / len(df['algorithm'].unique()) ratios = [] labels = [] weights = [] hist_algorithms = ['prim', 'khuller'] algorithm_labels = {'prim': 'Karger', 'khuller': 'Khuller'} sns.set() pylab.figure() for algorithm, group in df.groupby('algorithm'): print algorithm comparisons = group['comparisons'].sum() dominated = group['dominated'].sum() print float(dominated) / float( comparisons), "(", dominated, "/", comparisons, ")" print binom_test(dominated, comparisons) group = group.groupby('points', as_index=False).agg(pylab.mean) pylab.plot(group['points'], group['runtime'], label=algorithm) ratio = group['cost ratio'] ratio = ratio[~pylab.isnan(ratio)] ratio = ratio - 1 print "cost comparisons", len(ratio) print "cost ratio", pylab.mean(ratio), "+/-", pylab.std(ratio, ddof=1) if algorithm in hist_algorithms: ratios.append(ratio) labels.append(algorithm_labels[algorithm]) weight = pylab.ones_like(ratio) / float(len(ratio)) weights.append(weight) pylab.legend(loc=2) pylab.xlabel('number of points') pylab.ylabel('rumtime (minutes)') pylab.savefig('test_runtimes/runtimes.pdf', format='pdf') pylab.close() pylab.figure() pylab.hist(ratios, label=labels, weights=weights) pylab.xlabel('percent better/worse than Steiner', size=20) pylab.ylabel('proportion', size=20) pylab.legend() ax = pylab.gca() pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text pylab.tight_layout() pylab.savefig('test_runtimes/cost_ratios_hist.pdf', format='pdf') pylab.close()
def plot_funnel(pi_true, sigma_str): n = pl.exp(mc.rnormal(10, 2**-2, size=10000)) sigma = float(sigma_str)*pl.ones_like(n) p = pi_true*pl.ones_like(n) oln = rate_model.offset_log_normal('funnel', p, sigma, p, pl.sqrt(p*(1-p)/n)) r = oln['p_pred'].value pl.vlines([pi_true], .1*n.min(), 10*n.max(), linewidth=2, linestyle='-', color='w', zorder=9) pl.vlines([pi_true], .1*n.min(), 10*n.max(), linewidth=1, linestyle='--', color='black', zorder=10) pl.plot(r, n, 'ko', mew=0, alpha=.25) pl.semilogy(schiz['r'], schiz['n'], 'ks', mew=1, mec='white', ms=4, label='Observed values') pl.xlabel('Rate (per PY)') pl.ylabel('Study size (PY)') pl.xticks([0, .005, .01]) pl.axis([-.0001, .0101, 50., 15000000]) pl.title(r'$\sigma = %s$'%sigma_str)
def graph_stats(df): G = make_graph(df) print G.number_of_nodes(), "nodes" print G.number_of_edges(), "edges" degrees = [] for u in G.nodes_iter(): degrees.append(G.degree(u)) bins = range(max(degrees) + 1) weights = pylab.ones_like(degrees) / float(len(degrees)) pylab.figure() pylab.hist(degrees, bins=bins, weights=weights) pylab.savefig('repeatability/figs/degree_dist.pdf', format='pdf') print "transitivity", nx.transitivity(G) print "average clustering", nx.average_clustering(G) shortest_path_lengths = nx.shortest_path_length(G, weight=None) visited = set() path_lens = [] for source in shortest_path_lengths: for target in shortest_path_lengths[source]: if source == target: continue if tuple(sorted((source, target))) in visited: continue path_lens.append(shortest_path_lengths[source][target]) visited.add(tuple(sorted((source, target)))) print "mean shortest path len", pylab.mean(path_lens) print "diameter", max(path_lens) weights = pylab.ones_like(path_lens) / float(len(path_lens)) pylab.figure() pylab.hist(path_lens, weights=weights) #pylab.xlabel('shortest path length (mm)') pylab.savefig('repeatability/figs/path_lengths.pdf', format='pdf') pylab.close()
def chebspace(a, b, n, returnWeights=False): """Chebyshev nodes for given degree n""" apb = 0.5 * (a + b) bma = 0.5 * (b - a) cs = apb - bma * cos(arange(n) * pi / (n - 1)) # ensure that endpoints are exact cs[0] = a cs[-1] = b if returnWeights: weights = ones_like(cs) weights[::2] = -1 weights[0] /= 2 weights[-1] /= 2 return cs, weights else: return cs
def chebspace(a, b, n, returnWeights=False): """Chebyshev nodes for given degree n""" apb = 0.5 * (a + b) bma = 0.5 * (b - a) cs = apb - bma * cos(arange(n) * pi / (n-1)) # ensure that endpoints are exact cs[0] = a cs[-1] = b if returnWeights: weights = ones_like(cs) weights[::2] = -1 weights[0] /= 2 weights[-1] /= 2 return cs, weights else: return cs
def difficulty_hist(df): difficulties = [] weights = [] labels = [] pylab.figure() for name, group in df.groupby('Different plant'): difficulty = group['Junction difficulty'] print name print pylab.mean(difficulty), "pm", pylab.std(difficulty, ddof=1) weight = pylab.ones_like(difficulty) / float(len(difficulty)) difficulties.append(difficulty) weights.append(weight) labels.append(name + ' plant') pylab.hist(difficulties, bins=[1, 2, 3, 4, 5], label=labels, weights=weights) pylab.xlabel('edge difficulty') pylab.ylabel('frequency') pylab.legend() pylab.savefig('repeatability/figs/repeatability.pdf', format='pdf') pylab.close()
def test_age_pattern_model_sim(): # simulate normal data a = pl.arange(0, 100, 5) pi_true = .0001 * (a * (100. - a) + 100.) sigma_true = .025*pl.ones_like(pi_true) p = pl.maximum(0., mc.rnormal(pi_true, 1./sigma_true**2.)) # create model and priors vars = {} vars.update(age_pattern.age_pattern('test', ages=pl.arange(101), knots=pl.arange(0,101,5), smoothing=.1)) vars['pi'] = mc.Lambda('pi', lambda mu=vars['mu_age'], a=a: mu[a]) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2)
def test_covariate_model_sim_w_hierarchy(): n = 50 # setup hierarchy hierarchy, output_template = data_simulation.small_output() # simulate normal data area_list = pl.array(['all', 'USA', 'CAN']) area = area_list[mc.rcategorical([.3, .3, .4], n)] sex_list = pl.array(['male', 'female', 'total']) sex = sex_list[mc.rcategorical([.3, .3, .4], n)] year = pl.array(mc.runiform(1990, 2010, n), dtype=int) alpha_true = dict(all=0., USA=.1, CAN=-.2) pi_true = pl.exp([alpha_true[a] for a in area]) sigma_true = .05 * pl.ones_like(pi_true) p = mc.rnormal(pi_true, 1. / sigma_true**2.) model = data.ModelData() model.input_data = pandas.DataFrame( dict(value=p, area=area, sex=sex, year_start=year, year_end=year)) model.hierarchy, model.output_template = hierarchy, output_template # create model and priors vars = {} vars.update( covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model, 'all', 'total', 'all')) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2) assert 'sex' not in vars['U'] assert 'x_sex' in vars['X'] assert len(vars['beta']) == 1
def generate_data(N, delta_true, pi_true, heterogeneity, bias, sigma_prior): a = pl.arange(0, 101, 1) pi_age_true = pi_true(a) model = data_simulation.simple_model(N) model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10) model.parameters['p']['smoothness'] = dict(amount='Moderately') model.parameters['p']['heterogeneity'] = heterogeneity age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int) age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int) age_weights = pl.ones_like(a) sum_pi_wt = pl.cumsum(pi_age_true*age_weights) sum_wt = pl.cumsum(age_weights) p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] - sum_wt[age_start]) # correct cases where age_start == age_end i = age_start == age_end if pl.any(i): p[i] = pi_age_true[age_start[i]] n = mc.runiform(10000, 100000, size=N) model.input_data['age_start'] = age_start model.input_data['age_end'] = age_end model.input_data['effective_sample_size'] = n model.input_data['true'] = p model.input_data['value'] = mc.rnegative_binomial(n*p, delta_true*n*p) / n * pl.exp(bias) emp_priors = {} emp_priors['p', 'mu'] = pi_age_true emp_priors['p', 'sigma'] = sigma_prior*pi_age_true model.emp_priors = emp_priors model.a = a model.pi_age_true = pi_age_true model.delta_true = delta_true return model
def test_covariate_model_sim_w_hierarchy(): n = 50 # setup hierarchy hierarchy, output_template = data_simulation.small_output() # simulate normal data area_list = pl.array(['all', 'USA', 'CAN']) area = area_list[mc.rcategorical([.3, .3, .4], n)] sex_list = pl.array(['male', 'female', 'total']) sex = sex_list[mc.rcategorical([.3, .3, .4], n)] year = pl.array(mc.runiform(1990, 2010, n), dtype=int) alpha_true = dict(all=0., USA=.1, CAN=-.2) pi_true = pl.exp([alpha_true[a] for a in area]) sigma_true = .05*pl.ones_like(pi_true) p = mc.rnormal(pi_true, 1./sigma_true**2.) model = data.ModelData() model.input_data = pandas.DataFrame(dict(value=p, area=area, sex=sex, year_start=year, year_end=year)) model.hierarchy, model.output_template = hierarchy, output_template # create model and priors vars = {} vars.update(covariate_model.mean_covariate_model('test', 1, model.input_data, {}, model, 'all', 'total', 'all')) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2) assert 'sex' not in vars['U'] assert 'x_sex' in vars['X'] assert len(vars['beta']) == 1
def test_age_pattern_model_sim(): # simulate normal data a = pl.arange(0, 100, 5) pi_true = .0001 * (a * (100. - a) + 100.) sigma_true = .025 * pl.ones_like(pi_true) p = pl.maximum(0., mc.rnormal(pi_true, 1. / sigma_true**2.)) # create model and priors vars = {} vars.update( age_pattern.age_pattern('test', ages=pl.arange(101), knots=pl.arange(0, 101, 5), smoothing=.1)) vars['pi'] = mc.Lambda('pi', lambda mu=vars['mu_age'], a=a: mu[a]) vars.update(rate_model.normal_model('test', vars['pi'], 0., p, sigma_true)) # fit model m = mc.MCMC(vars) m.sample(2)
def sim(vx0,y0,L01,a0,k1,dE,m,ygrd,t,fs,steps): # function for using fmin # def optfunc_dE(par): # return abs(-.5*k1*c**2+.5*(k1+par)*(c-(c*par/(k1+par)))**2-dE) # function for using fsolve def optfunc_dE(par, spring_par): k1 = spring_par['k'] c = spring_par['c'] return abs(-.5*k1*c**2+.5*(k1+par)*(c-(c*par/(k1+par)))**2-dE) g = -9.81 # gravitational force betaTO_sim = zeros(steps) xF=0 print 'x' x_sim = [0] y_sim = [0] vx_sim = [0] vy_sim = [0] t_sim = [0] k2 = 0 L02 = 0 skip_all = False st_lo_idx = -1 # leg orientation xL = L01 * cos(a0 * pi/180.) yL = L01 * sin(a0 * pi/180.) # first ground contact t_gc = sqrt(2.*(L01*sin(a0*pi/180.)-y0)/g) if isnan(t_gc) == True: skip_all = True if skip_all == False: x_gc = vx0*t_gc y_gc = .5*g*t_gc**2 + y0 # system parameter for first step s = zeros(4) s[0] = 0 s[1] = y0 s[2] = vx0 s[3] = 0 # definitions for calculation end_sim = False res = zeros([1,4]) res_app = [] t_v = [0] skip_stance = False # start loop while end_sim == False: st_lo_idx += 1 if st_lo_idx != 0: # if not first step t_gc = (-res[-1,3]/g) + \ sqrt( ((res[-1,3]**2)/(g**2)) - \ ((2*res[-1,1])/g-(2*L01*sin(a0*pi/180)/g)) ) if isnan(t_gc) == False and isinf(t_gc) == False: y_gc = 0.5*g*t_gc**2 + res[-1,3]*t_gc + res[-1,1] t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1) #print len(t_vec) s[0] = res[-1,2]*t_gc + res[-1,0] s[1] = y_gc s[2] = res[-1,2] s[3] = g*t_gc +res[-1,3] else: t_gc = 1 t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1) end_sim = True skip_stance = True #break vx = zeros_like(t_vec) vy = zeros_like(t_vec) x = zeros_like(t_vec) y = zeros_like(t_vec) vx[0] = res[-1,2] vy[0] = res[-1,3] x[0] = res[-1,0] y[0] = res[-1,1] else: # if first step t_vec = linspace(0,t_gc,ceil(t_gc*fs)+1) #print t_vec vx = zeros_like(t_vec) vy = zeros_like(t_vec) x = zeros_like(t_vec) y = zeros_like(t_vec) #print vx vx[0] = s[2] vy[0] = s[3] x[0] = s[0] y[0] = s[1] ### flight phase x[1:] = vx[0]*t_vec[1:] + x[0] y[1:] = .5*g*(t_vec[1:]**2)+vy[0]*t_vec[1:] + y[0] vy[1:] = g*t_vec[1:] + vy[0] vx[1:] = vx[0]*ones_like(t_vec[1:]) # end at apex if st_lo_idx == steps or skip_stance == True: tapex = vy[0]/-g #print 'y0=',y0,'vy0=',vy[0],' tapex=',tapex tvec_apex = linspace(0,tapex,ceil(tapex*fs)+1) vyapex = g*tvec_apex+vy[0] #print 'vyapex=',vyapex yapex = .5*g*(tvec_apex**2) + vyapex[0]*tvec_apex + y[0] xapex = vx[0] * tvec_apex + x[0] vxapex = vx[0] * ones_like(tvec_apex) #print 'lvy:',len(vyapex),' lvx:',len(vxapex) x_sim = append(x_sim,xapex[1:]) y_sim = append(y_sim,yapex[1:]) vx_sim = append(vx_sim,vxapex[1:]) vy_sim = append(vy_sim,vyapex[1:]) t_sim = append(t_sim,tvec_apex[1:]+t_sim[-1]) skip_stance = True """ for kk in arange(2,len(y)): if y[kk-2]<y[kk-1] and y[kk-1]>y[kk]: end_sim = True x_sim = append(x_sim,x[1:kk]) y_sim = append(y_sim,y[1:kk]) vx_sim = append(vx_sim,vx[1:kk]) vy_sim = append(vy_sim,vy[1:kk]) t_sim = append(t_sim,t_vec[1:kk]+t_sim[-1]) """ ### stance phase if skip_stance == False: # system parameter if st_lo_idx == 0: # if first step s[0] = x_gc # x s[1] = y_gc # y s[2] = vx0 # vx s[3] = g*t_gc + 0 # vy xF = s[0]+xL yF = ygrd parameter = {'k':k1,'L0':L01,'m':m,'g':g,'xF':xF,'yF':yF} calc_length = 0.1*fs find_pos = False round_prec = 11 # precision of rounding #print 'round_prec = ', round_prec step_size = 1./fs res = zeros([1,4]) t_v = [0] res_app = False skip = False L0 = L01 FinMin = False # start integration of stance phase while find_pos == False: t_vint = (step_size) * arange(calc_length) res_int = odeint(s_dot,s,t_vint, args=(parameter,), rtol=1e-11) # stop condition vx or y < 0 if res_int[-1,1] < 0 or res_int[-1,2] < 0: find_pos = True end_sim = True skip = True # if ymin: calculation of new k and L for energy changes for idx in arange(1,len(res_int[:,1])-1): if res_int[idx,1]<res_int[idx-1,1] and res_int[idx,1]<res_int[idx+1,1]: res_int=delete(res_int,s_[idx+1::],0) t_vint = delete(t_vint,s_[idx+1::],0) c= L0-(sqrt(res_int[-1,1]**2 + (xF-res_int[-1,0])**2)) #dk = fmin(optfunc_dE,0,full_output=1,disp=0,xtol=1e-12) spring_par = {'k':k1,'c':c} dk = fsolve(optfunc_dE,x0=[0],args=(spring_par,)) #print '###### dk ####',dk # if not find a minimum stop loop --> no step #if dk[-1] <> 0: # find_pos = True # end_sim = True # skip = True # raise ValueError, 'no energy management possible' # break k2 = k1+dk[0] L02 = L01 - c*dk[0]/(k1+dk[0]) #print 'delta L = ',L0 - L02, ' delta k = ', dk[0] L0 = L02 parameter = {'k':k2,'L0':L02,'m':m,'g':g,'xF':xF,'yF':yF} FinMin = True break # if not ymin: start one before to avoid ignoring while change if FinMin == False: res_int = res_int[:-1] t_vint = t_vint[:-1] # virtual leg after integration L_virt = sqrt(res_int[-1,1]**2 + (xF-res_int[-1,0])**2) if L_virt < L0 and skip == False: s[0] = res_int[-1,0] s[1] = res_int[-1,1] s[2] = res_int[-1,2] s[3] = res_int[-1,3] elif L_virt >= L0 and skip == False: for jj in arange(1,len(res_int)): L_virt_1 = sqrt(res_int[jj,1]**2 + abs(xF-res_int[jj,0])**2) L_virt_2 = sqrt(res_int[jj-1,1]**2 + abs(xF-res_int[jj-1,0])**2) if round(L_virt_2,round_prec) == round(L0,round_prec): find_pos = True res_int = res_int[0:jj,:] res_app = True t_vint = t_vint[0:jj] if res_int[-1,3] <=0: end_sim = True break elif L_virt_2 < L0 and L_virt_1 > L0: s[0] = res_int[jj-1,0] s[1] = res_int[jj-1,1] s[2] = res_int[jj-1,2] s[3] = res_int[jj-1,3] res_int = res_int[0:jj,:] res_app = True t_vint = t_vint[0:jj] step_size = step_size / 10. break # append arrays if res_app == False: res = res_int t_v = t_vint res_app = True else: res = append(res,res_int[1::],0) t_v = append(t_v,t_vint[1::]+t_v[-1]) # take of angle #print st_lo_idx betaTO_sim[st_lo_idx] = arctan(res[-1,1]/(res[-1,0]-xF))*180./pi # append arrays x_sim = append(x_sim,append(x,res[:,0])) y_sim = append(y_sim,append(y,res[:,1])) vx_sim = append(vx_sim,append(vx,res[:,2])) vy_sim = append(vy_sim,append(vy,res[:,3])) t_sim = append(t_sim,append(t_vec,t_v+t_gc)+t_sim[-1]) else: #skip_stance == True because of ending at apex end_sim = True # cut array x_sim = x_sim[1::] y_sim = y_sim[1::] vx_sim = vx_sim[1::] vy_sim = vy_sim[1::] t_sim = t_sim[1::] #print 'vor return' return x_sim,y_sim,vx_sim,vy_sim,t_sim,st_lo_idx,betaTO_sim,xF,k2,L02
def fit_world(id, fast_fit=False, zero_re=True, alt_prior=False, global_heterogeneity='Slightly'): """ Fit consistent for all data in world Parameters ---------- id : int The model id number for the job to fit Example ------- >>> import fit_world >>> dm = fit_world.dismod3.load_disease_model(1234) >>> fit_world.fit_world(dm) """ dir = dismod3.settings.JOB_WORKING_DIR % id ## load the model from disk or from web import simplejson as json import data reload(data) try: model = data.ModelData.load(dir) print 'loaded data from new format from %s' % dir dm = dismod3.load_disease_model(id) except (IOError, AssertionError): dm = dismod3.load_disease_model(id) model = data.ModelData.from_gbd_jsons(json.loads(dm.to_json())) try: model.save(dir) print 'loaded data from json, saved in new format for next time in %s' % dir except IOError: print 'loaded data from json, failed to save in new format' ## next block fills in missing covariates with zero for col in model.input_data.columns: if col.startswith('x_'): model.input_data[col] = model.input_data[col].fillna(0.) # also fill all covariates missing in output template with zeros model.output_template = model.output_template.fillna(0) # set all heterogeneity priors to Slightly for the global fit for t in model.parameters: if 'heterogeneity' in model.parameters[t]: model.parameters[t]['heterogeneity'] = global_heterogeneity ### For testing: ## speed up computation by reducing number of knots ## for t in 'irf': ## model.parameters[t]['parameter_age_mesh'] = [0, 100] model.vars += dismod3.ism.consistent(model, reference_area='all', reference_sex='total', reference_year='all', priors={}, zero_re=zero_re) ## fit model to data if fast_fit: dm.map, dm.mcmc = dismod3.fit.fit_consistent(model, 105, 0, 1, 100) else: dm.map, dm.mcmc = dismod3.fit.fit_consistent(model, iter=50000, burn=10000, thin=40, tune_interval=1000, verbose=True) dm.model = model # borrow strength to inform sigma_alpha between rate types post-hoc types_with_re = ['rr', 'f', 'i', 'm', 'smr', 'p', 'r', 'pf', 'm_with', 'X'] ## first calculate sigma_alpha_bar from posterior draws from each alpha alpha_vals = [] for type in types_with_re: if 'alpha' in model.vars[type]: for alpha_i in model.vars[type]['alpha']: alpha_vals += [a for a in alpha_i.trace() if a != 0] # remove zeros because areas with no siblings are included for convenience but are pinned to zero ## then blend sigma_alpha_i and sigma_alpha_bar for each sigma_alpha_i if len(alpha_vals) > 0: sigma_alpha_bar = pl.std(alpha_vals) for type in types_with_re: if 'sigma_alpha' in model.vars[type]: for sigma_alpha_i in model.vars[type]['sigma_alpha']: cur_val = sigma_alpha_i.trace() sigma_alpha_i.trace._trace[0] = (cur_val + sigma_alpha_bar) * pl.ones_like(sigma_alpha_i.trace._trace[0]) for t in 'p i r f rr pf m_with'.split(): param_type = dict(i='incidence', r='remission', f='excess-mortality', p='prevalence', rr='relative-risk', pf='prevalence_x_excess-mortality', m_with='mortality')[t] #graphics.plot_one_type(model, model.vars[t], {}, t) for a in [dismod3.utils.clean(a) for a in dismod3.settings.gbd_regions]: print 'generating empirical prior for %s' % a for s in dismod3.settings.gbd_sexes: for y in dismod3.settings.gbd_years: key = dismod3.utils.gbd_key_for(param_type, a, y, s) if t in model.parameters and 'level_bounds' in model.parameters[t]: lower=model.parameters[t]['level_bounds']['lower'] upper=model.parameters[t]['level_bounds']['upper'] else: lower=0 upper=pl.inf emp_priors = covariate_model.predict_for(model, model.parameters.get(t, {}), 'all', 'total', 'all', a, dismod3.utils.clean(s), int(y), alt_prior, model.vars[t], lower, upper) dm.set_mcmc('emp_prior_mean', key, emp_priors.mean(0)) if 'eta' in model.vars[t]: N,A = emp_priors.shape # N samples, for A age groups delta_trace = pl.transpose([pl.exp(model.vars[t]['eta'].trace()) for _ in range(A)]) # shape delta matrix to match prediction matrix emp_prior_std = pl.sqrt(emp_priors.var(0) + (emp_priors**2 / delta_trace).mean(0)) else: emp_prior_std = emp_priors.std(0) dm.set_mcmc('emp_prior_std', key, emp_prior_std) from fit_emp_prior import store_effect_coefficients store_effect_coefficients(dm, model.vars[t], param_type) if 'p_pred' in model.vars[t]: graphics.plot_one_ppc(model, t) pl.savefig(dir + '/prior-%s-ppc.png'%param_type) if 'p_pred' in model.vars[t] or 'lb' in model.vars[t]: graphics.plot_one_effects(model, t) pl.savefig(dir + '/prior-%s-effects.png'%param_type) for t in 'i r f p rr pf X m_with smr'.split(): fname = dir + '/empirical_priors/data-%s.csv'%t print 'saving tables for', t, 'to', fname if 'data' in model.vars[t] and 'p_pred' in model.vars[t]: stats = model.vars[t]['p_pred'].stats(batches=5) model.vars[t]['data']['mu_pred'] = stats['mean'] model.vars[t]['data']['sigma_pred'] = stats['standard deviation'] stats = model.vars[t]['pi'].stats(batches=5) model.vars[t]['data']['mc_error'] = stats['mc error'] model.vars[t]['data']['residual'] = model.vars[t]['data']['value'] - model.vars[t]['data']['mu_pred'] model.vars[t]['data']['abs_residual'] = pl.absolute(model.vars[t]['data']['residual']) #if 'delta' in model.vars[t]: # model.vars[t]['data']['logp'] = [mc.negative_binomial_like(n*p_obs, n*p_pred, n*p_pred*d) for n, p_obs, p_pred, d \ # in zip(model.vars[t]['data']['effective_sample_size'], # model.vars[t]['data']['value'], # model.vars[t]['data']['mu_pred'], # pl.atleast_1d(model.vars[t]['delta'].stats()['mean']))] model.vars[t]['data'].to_csv(fname) graphics.plot_fit(model) pl.savefig(dir + '/prior.png') graphics.plot_acorr(model) pl.savefig(dir + '/prior-convergence.png') graphics.plot_trace(model) pl.savefig(dir + '/prior-trace.png') # save results (do this last, because it removes things from the disease model that plotting function, etc, might need try: dm.save('dm-%d-prior-%s.json' % (dm.id, 'all')) except IOError, e: print e
pl.title(r'$\sigma = %s$'%sigma_str) pl.figure(figsize=(11, 8.5), dpi=120) pl.subplots_adjust(wspace=.4) pl.subplot(2,2,1) plot_funnel(.004, '0.5') pl.subplot(2,2,2) plot_funnel(.004, '0.1') pl.subplot(2,1,2) r = pl.array(schiz['r']) n = pl.array(schiz['n']) pi = mc.Uniform('pi', 0, 1, value=.001) sigma = mc.Uniform('sigma', 0, 100, value=.0001) oln = rate_model.offset_log_normal('funnel', pi*pl.ones_like(n), sigma*pl.ones_like(n), r, pl.sqrt(r*(1-r)/n)) mcmc = mc.MCMC([pi, sigma, oln]) mcmc.sample(20000, 10000, 10, verbose=False, progress_bar=False) sorted_indices = r.argsort().argsort() jitter = mc.rnormal(0, .1**-2, len(oln['p_pred'].trace())) for i,s_i in enumerate(sorted_indices): pl.plot(s_i+jitter, oln['p_pred'].trace()[:, i], 'ko', mew=0, alpha=.25, zorder=-99) pl.errorbar(sorted_indices, r, yerr=1.96*pl.sqrt(r*(1-r)/n), fmt='ws', mew=1, mec='white', ms=5, elinewidth=3, capsize=0) pl.errorbar(sorted_indices, r, yerr=1.96*pl.sqrt(r*(1-r)/n), fmt='ks', mew=1, mec='white', ms=5) pl.xticks([]) pl.ylabel('Rate (per PY)') pl.axis([-.5, 15.5,-.0001,.0121])
def greedy_eval_stats(model='uniform'): df = pd.read_csv('greedy_eval_%s.csv' % model, skipinitialspace=True) df['success'] = df['cost ratio'] == 0 df['error'] = df['cost ratio'] < 0 mcost_ratios = [] scost_ratios = [] cost_ratios = [] time_ratios = [] algorithm_labels = {'brute force' : 'Brute force', 'steiner' : 'Greedy', 'prim' : 'Karger', 'khuller' : 'Khuller'} labels = [] for algorithm, group in df.groupby('algorithm'): print algorithm alpha = group['alpha'] print len(alpha) / len(alpha.unique()), "point sets", len(alpha), "trials" print "success rate", pylab.mean(group['success']) print "error rate", pylab.mean(group['error']) print "dominated", pylab.mean(group['dominated']) mcost_ratio = group['mcost ratio'] scost_ratio = group['scost ratio'] cost_ratio = group['cost ratio'] time_ratio = group['time ratio'] print "mcost ratio", pylab.nanmean(mcost_ratio), "+/-", pylab.nanstd(mcost_ratio, ddof=1) print "scost ratio", pylab.nanmean(scost_ratio), "+/-", pylab.nanstd(scost_ratio, ddof=1) print "cost ratio", pylab.nanmean(cost_ratio), "+/-", pylab.nanstd(cost_ratio, ddof=1) print "time ratio", pylab.nanmean(time_ratio), "+/-", pylab.nanstd(time_ratio, ddof=1) print "max mcost ratio", pylab.nanmax(mcost_ratio) print "max scost ratio", pylab.nanmax(scost_ratio) print "max cost ratio", pylab.nanmax(cost_ratio) print "max time ratio", pylab.nanmax(time_ratio) mcost_ratios.append(mcost_ratio) scost_ratios.append(scost_ratio) cost_ratios.append(cost_ratio) time_ratios.append(time_ratio) labels.append(algorithm_labels[algorithm]) for points, points_group in group.groupby('points'): print points, pylab.mean(points_group['time']) sns.set() axis_text = {'mcost' : 'wiring cost', 'scost' : 'conduction delay', 'cost' : 'Pareto cost', 'time' : 'runtime'} for ratios, fname in zip([mcost_ratios, scost_ratios, cost_ratios, time_ratios], ['mcost', 'scost', 'cost', 'time']): pylab.figure() x = [] weights = [] for ratio, label in zip(ratios, labels): ratio = pylab.array(ratio) ratio = ratio[~pylab.isnan(ratio)] x.append(ratio) wt = pylab.ones_like(ratio) / float(len(ratio)) weights.append(wt) pylab.hist(x, weights=weights, label=labels) pylab.legend() ax = pylab.gca() pylab.setp(ax.get_legend().get_texts(), fontsize=20) # for legend text ax_text = axis_text[fname] pylab.xlabel('%s (percent higher/lower than %s)' % (ax_text, BASELINE), size=20) pylab.ylabel('proportion', size=20) pylab.tight_layout() pylab.savefig('greedy_eval/%s_ratios_hist.pdf' % fname, format='pdf') pylab.close()
def SLIP_step(IC, SLIP_params, sim_params=[]): """ simulates the SLIP IC: initial state vector, containing y0, vx0 (x0 is assumed to be 0; vy0 = 0 (apex); also ground level = 0) SLIP_params: k L0 m alpha dE: energy change in "midstance" by changing k and L0 g: gravity (negative! should be ~ -9.81 for SI) """ alpha = SLIP_params['alpha'] k = SLIP_params['k'] L0 = SLIP_params['L0'] dE = SLIP_params['dE'] g = SLIP_params['g'] m = SLIP_params['m'] y0 = IC[0] vx0 = IC[1] if g >= 0: raise ValueError, "gravity points into wrong direction!" # concatenate state vector of four elements: # (1) time to touchdown # (2) time to vy = 0 # (3) time to takeoff # (4) time to apex # (1) and (4) are analytically described y_land = L0 * sin(alpha) if y0 < y_land: raise ValueError, "invalid starting condition" # before starting, define the model: def SLIP_ode(y, t, params): """ defines the ODE of the SLIP, under stance condition state: [x y vx vy] params: {'L0' : leg rest length 'x0' : leg touchdown position 'k' : spring stiffness 'm' : mass} """ dy0 = y[2] dy1 = y[3] L = sqrt((y[0] - params['xF'])**2 + y[1]**2) F = params['k'] * (params['L0'] - L) Fx = F * (y[0] - params['xF']) / L Fy = F * y[1] / L dy2 = Fx / m dy3 = Fy / m + params['g'] return hstack([dy0, dy1, dy2, dy3]) def sim_until(IC, params, stop_fcn, tmax=2.): """ simulated the SLIP_ode until stop_fcn has a zero-crossing includes a refinement of the time at this instant stop_fcn must be a function of the system state, e.g. stop_fcn(IC) must exist this function is especially adapted to the SLIP state, so it uses dot(x1) = x3, dot(x2) = x4 tmax: maximal simulation time [s] """ init_sign = sign(stop_fcn(IC)) #1st: evaluate a certain fraction tvec_0 = .0001 * arange(50) sim_results = [] sim_tvecs = [] newIC = IC sim_results.append( odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12)) sim_tvecs.append(tvec_0) check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]] t_tot = 0. while min(check_vec) > 0: newIC = sim_results[-1][-1, :] sim_results.append( odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12)) sim_tvecs.append(tvec_0) check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]] t_tot += tvec_0[-1] # time exceeded or ground hit if t_tot > tmax or min(sim_results[-1][:, 1] < 0): raise SimFailError, "simulation failed" # now: zero-crossing detected # -> refine! minidx = find(array(check_vec) < 0)[0] if minidx == 0: # this should not happen because the first value in # check_vec should be BEFORE the zero_crossing by # construction raise ValueError, "ERROR: this should not happen!" # refine simulation by factor 50, but only for two # adjacent original time frames newIC = sim_results[-1][minidx - 1, :] sim_results[-1] = sim_results[-1][:minidx, :] sim_tvecs[-1] = sim_tvecs[-1][:minidx] # avoid that last position can be the zero-crossing n_refine = 10000 tvec_0 = linspace(tvec_0[0], tvec_0[1] + 2. / n_refine, n_refine + 2) sim_results.append( odeint(SLIP_ode, newIC, tvec_0, args=(params, ), rtol=1e-12)) sim_tvecs.append(tvec_0) # linearly interpolate to zero check_vec = [init_sign * stop_fcn(x) for x in sim_results[-1]] minidx = find(array(check_vec) < 0)[0] if minidx == 0: # this should not happen because the first value in # check_vec should be BEFORE the zero_crossing by # construction raise ValueError, "ERROR: this should not happen! (2)" # compute location of zero-crossing y0 = sim_results[-1][minidx - 1, :] y1 = sim_results[-1][minidx, :] fcn0 = stop_fcn(y0) fcn1 = stop_fcn(y1) t0 = tvec_0[minidx - 1] t1 = tvec_0[minidx] t_zero = t0 - (t1 - t0) * fcn0 / (fcn1 - fcn0) # cut last simulation result and replace last values # by interpolated values sim_results[-1] = sim_results[-1][:minidx + 1, :] sim_tvecs[-1] = sim_tvecs[-1][:minidx + 1] for coord in arange(sim_results[-1].shape[1]): sim_results[-1][-1, coord] = interp( t_zero, [t0, t1], [sim_results[-1][-2, coord], sim_results[-1][-1, coord]]) sim_tvecs[-1][-1] = t_zero #newIC = sim_results[-1][minidx-1,:] #sim_results[-1] = sim_results[-1][:minidx,:] #sim_tvecs[-1] = sim_tvecs[-1][:minidx] #tvec_0 = linspace(tvec_0[0],tvec_0[1],100) #sim_results.append ( odeint(SLIP_ode, newIC, tvec_0, # args=(params,),rtol=1e-9)) #sim_tvecs.append(tvec_0) # concatenate lists sim_data = vstack( [x[:-1, :] for x in sim_results[:-1] if x.shape[0] > 1] + [ sim_results[-1], ]) sim_time = [ sim_tvecs[0], ] for idx in arange(1, len(sim_tvecs)): sim_time.append(sim_tvecs[idx] + sim_time[-1][-1]) sim_time = hstack([x[:-1] for x in sim_time[:-1]] + [ sim_time[-1], ]) return sim_data, sim_time # Section 1: time to touchdown # TODO: make sampling frequency regular t_flight1 = sqrt(-2. * (y0 - y_land) / g) #t_flight = sqrt() tvec_flight1 = .01 * arange(t_flight1 * 100.) vy_flight1 = tvec_flight1 * g y_flight1 = y0 + .5 * g * (tvec_flight1**2) x_flight1 = vx0 * tvec_flight1 vx_flight1 = vx0 * ones_like(tvec_flight1) # Section 2: time to vy = 0 # approach: calculate forward -> estimate interval of # zero position of vy -> refine simulation in that interval # until a point with vy sufficiently close to zero is in the # resulting vector params = { 'L0': L0, 'xF': t_flight1 * vx0 + L0 * cos(alpha), 'k': k, 'm': m, 'g': g } IC = array([t_flight1 * vx0, y_land, vx0, t_flight1 * g]) # initial guess: L0*cos(alpha)/vx0 #t_sim1 = L0*cos(alpha)/vx0 # TODO: implement sim_fail check! sim_fail = False try: sim_phase2, t_phase2 = sim_until(IC, params, lambda x: x[3]) t_phase2 += t_flight1 except SimFailError: print 'simulation aborted (phase 2)\n' sim_fail = True # Phase 3: if not sim_fail: L = sqrt(sim_phase2[-1, 1]**2 + (sim_phase2[-1, 0] - params['xF'])**2) dk, dL = dk_dL(L0, k, L, dE) params2 = deepcopy(params) params2['k'] += dk params2['L0'] += dL IC = sim_phase2[-1, :] compression = (lambda x: sqrt( (x[0] - params2['xF'])**2 + x[1]**2) - params2['L0']) #print ('L:', L, 'dk', dk, 'dL', dL, 'dE', dE, '\ncompression:', compression(IC), # 'IC', IC) try: sim_phase3, t_phase3 = sim_until(IC, params2, compression) sim_phase3 = sim_phase3[1:, :] t_phase3 = t_phase3[1:] + t_phase2[-1] except SimFailError: print 'simulation aborted (phase 3)\n' sim_fail = True # Phase 4: if not sim_fail: # time to apex # TODO: make sampling frequency regular vy_liftoff = sim_phase3[-1, 3] t_flight2 = -1. * vy_liftoff / g #t_flight = sqrt() tvec_flight2 = arange(t_flight2, 0, -.001)[::-1] vy_flight2 = tvec_flight2 * g + vy_liftoff y_flight2 = (sim_phase3[-1, 1] + vy_liftoff * tvec_flight2 + .5 * g * (tvec_flight2**2)) x_flight2 = sim_phase3[-1, 0] + sim_phase3[-1, 2] * tvec_flight2 vx_flight2 = sim_phase3[-1, 2] * ones_like(tvec_flight2) tvec_flight2 += t_phase3[-1] # todo: return data until error if sim_fail: return { 't': None, 'x': None, 'y': None, 'vx': None, 'vy': None, 'sim_fail': sim_fail, 'dk': None, 'dL': None } # finally: concatenate phases x_final = hstack( [x_flight1, sim_phase2[:, 0], sim_phase3[:, 0], x_flight2]) y_final = hstack( [y_flight1, sim_phase2[:, 1], sim_phase3[:, 1], y_flight2]) vx_final = hstack( [vx_flight1, sim_phase2[:, 2], sim_phase3[:, 2], vx_flight2]) vy_final = hstack( [vy_flight1, sim_phase2[:, 3], sim_phase3[:, 3], vy_flight2]) tvec_final = hstack([tvec_flight1, t_phase2, t_phase3, tvec_flight2]) return { 't': tvec_final, 'x': x_final, 'y': y_final, 'vx': vx_final, 'vy': vy_final, 'sim_fail': sim_fail, 'dk': dk, 'dL': dL, #'sim_res':sim_res, #'sim_phase2': sim_phase2_cut, #'t_phase2': t_phase2_cut }
def validate_age_integrating_model_sim(N=500, delta_true=.15, pi_true=quadratic): ## generate simulated data a = pl.arange(0, 101, 1) pi_age_true = pi_true(a) model = data_simulation.simple_model(N) #model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10) #model.parameters['p']['smoothness'] = dict(amount='Very') age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int) age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int) age_weights = pl.ones_like(a) sum_pi_wt = pl.cumsum(pi_age_true * age_weights) sum_wt = pl.cumsum(age_weights) p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] - sum_wt[age_start]) # correct cases where age_start == age_end i = age_start == age_end if pl.any(i): p[i] = pi_age_true[age_start[i]] n = mc.runiform(100, 10000, size=N) model.input_data['age_start'] = age_start model.input_data['age_end'] = age_end model.input_data['effective_sample_size'] = n model.input_data['true'] = p model.input_data['value'] = mc.rnegative_binomial(n * p, delta_true * n * p) / n ## Then fit the model and compare the estimates to the truth model.vars = {} model.vars['p'] = data_model.data_model('p', model, 'p', 'all', 'total', 'all', None, None, None) model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'], iter=10000, burn=5000, thin=25, tune_interval=100) graphics.plot_one_ppc(model.vars['p'], 'p') graphics.plot_convergence_diag(model.vars) graphics.plot_one_type(model, model.vars['p'], {}, 'p') pl.plot(a, pi_age_true, 'r:', label='Truth') pl.legend(fancybox=True, shadow=True, loc='upper left') pl.show() model.input_data['mu_pred'] = model.vars['p']['p_pred'].stats()['mean'] model.input_data['sigma_pred'] = model.vars['p']['p_pred'].stats( )['standard deviation'] data_simulation.add_quality_metrics(model.input_data) model.delta = pandas.DataFrame(dict(true=[delta_true])) model.delta['mu_pred'] = pl.exp(model.vars['p']['eta'].trace()).mean() model.delta['sigma_pred'] = pl.exp(model.vars['p']['eta'].trace()).std() data_simulation.add_quality_metrics(model.delta) print 'delta' print model.delta print '\ndata prediction bias: %.5f, MARE: %.3f, coverage: %.2f' % ( model.input_data['abs_err'].mean(), pl.median(pl.absolute(model.input_data['rel_err'].dropna())), model.input_data['covered?'].mean()) model.mu = pandas.DataFrame( dict(true=pi_age_true, mu_pred=model.vars['p']['mu_age'].stats()['mean'], sigma_pred=model.vars['p']['mu_age'].stats() ['standard deviation'])) data_simulation.add_quality_metrics(model.mu) model.results = dict(param=[], bias=[], mare=[], mae=[], pc=[]) data_simulation.add_to_results(model, 'delta') data_simulation.add_to_results(model, 'mu') data_simulation.add_to_results(model, 'input_data') model.results = pandas.DataFrame(model.results, columns='param bias mae mare pc'.split()) print model.results return model
'incidence': 'i', 'remission': 'r', 'excess-mortality': 'f' }[rate_type] for i, k_i in enumerate(model.vars[t]['knots']): model.vars[t]['gamma'][i].value = pl.log(pl.maximum(1.e-9, value[i])) def set_birth_prev(value): model.vars['logit_C0'].value = mc.logit(pl.maximum(1.e-9, value)) # <codecell> ### @export 'mental' set_rate('remission', pl.ones_like(ages) * .25) set_rate('incidence', pl.where((ages > 14) & (ages < 65), (65 - ages) * .001, 0.)) set_birth_prev(0) set_rate('excess-mortality', 1e-4 * pl.ones_like(ages)) book_graphics.plot_age_patterns(model, yticks=[0, .2, .4], panel='a') pl.savefig('forward-sim-mental.pdf') # <codecell> ### @export 'congenital' set_rate('remission', pl.zeros_like(ages)) set_rate('incidence', pl.zeros_like(ages)) set_birth_prev(.2) set_rate('excess-mortality', .5 * (ages / 100.)**2)
def SLIP_step(IC,SLIP_params,sim_params = []): """ simulates the SLIP IC: initial state vector, containing y0, vx0 (x0 is assumed to be 0; vy0 = 0 (apex); also ground level = 0) SLIP_params: k L0 m alpha dE: energy change in "midstance" by changing k and L0 g: gravity (negative! should be ~ -9.81 for SI) """ alpha = SLIP_params['alpha'] k = SLIP_params['k'] L0 = SLIP_params['L0'] dE = SLIP_params['dE'] g = SLIP_params['g'] m = SLIP_params['m'] y0 = IC[0] vx0 = IC[1] if g >= 0: raise ValueError, "gravity points into wrong direction!" # concatenate state vector of four elements: # (1) time to touchdown # (2) time to vy = 0 # (3) time to takeoff # (4) time to apex # (1) and (4) are analytically described y_land = L0*sin(alpha) if y0 < y_land: raise ValueError, "invalid starting condition" # before starting, define the model: def SLIP_ode(y,t,params): """ defines the ODE of the SLIP, under stance condition state: [x y vx vy] params: {'L0' : leg rest length 'x0' : leg touchdown position 'k' : spring stiffness 'm' : mass} """ dy0 = y[2] dy1 = y[3] L = sqrt((y[0]-params['xF'])**2 + y[1]**2) F = params['k']*(params['L0']-L) Fx = F*(y[0]-params['xF'])/L Fy = F*y[1]/L dy2 = Fx/m dy3 = Fy/m + params['g'] return hstack([dy0,dy1,dy2,dy3]) def sim_until(IC, params, stop_fcn, tmax = 2.): """ simulated the SLIP_ode until stop_fcn has a zero-crossing includes a refinement of the time at this instant stop_fcn must be a function of the system state, e.g. stop_fcn(IC) must exist this function is especially adapted to the SLIP state, so it uses dot(x1) = x3, dot(x2) = x4 tmax: maximal simulation time [s] """ init_sign = sign(stop_fcn(IC)) #1st: evaluate a certain fraction tvec_0 = .0001*arange(50) sim_results = [] sim_tvecs = [] newIC = IC sim_results.append (odeint(SLIP_ode,newIC,tvec_0, args=(params,),rtol=1e-12)) sim_tvecs.append(tvec_0) check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]] t_tot = 0. while min(check_vec) > 0: newIC = sim_results[-1][-1,:] sim_results.append ( odeint(SLIP_ode, newIC, tvec_0, args=(params,),rtol=1e-12)) sim_tvecs.append(tvec_0) check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]] t_tot += tvec_0[-1] # time exceeded or ground hit if t_tot > tmax or min(sim_results[-1][:,1] < 0): raise SimFailError, "simulation failed" # now: zero-crossing detected # -> refine! minidx = find(array(check_vec) < 0)[0] if minidx == 0: # this should not happen because the first value in # check_vec should be BEFORE the zero_crossing by # construction raise ValueError, "ERROR: this should not happen!" # refine simulation by factor 50, but only for two # adjacent original time frames newIC = sim_results[-1][minidx-1,:] sim_results[-1] = sim_results[-1][:minidx,:] sim_tvecs[-1] = sim_tvecs[-1][:minidx] # avoid that last position can be the zero-crossing n_refine = 10000 tvec_0 = linspace(tvec_0[0], tvec_0[1] + 2./n_refine, n_refine+2) sim_results.append ( odeint(SLIP_ode, newIC, tvec_0, args=(params,),rtol=1e-12)) sim_tvecs.append(tvec_0) # linearly interpolate to zero check_vec = [init_sign*stop_fcn(x) for x in sim_results[-1]] minidx = find(array(check_vec) < 0)[0] if minidx == 0: # this should not happen because the first value in # check_vec should be BEFORE the zero_crossing by # construction raise ValueError, "ERROR: this should not happen! (2)" # compute location of zero-crossing y0 = sim_results[-1][minidx-1,:] y1 = sim_results[-1][minidx,:] fcn0 = stop_fcn(y0) fcn1 = stop_fcn(y1) t0 = tvec_0[minidx-1] t1 = tvec_0[minidx] t_zero = t0 - (t1-t0)*fcn0/(fcn1 - fcn0) # cut last simulation result and replace last values # by interpolated values sim_results[-1] = sim_results[-1][:minidx+1,:] sim_tvecs[-1] = sim_tvecs[-1][:minidx+1] for coord in arange(sim_results[-1].shape[1]): sim_results[-1][-1,coord] = interp( t_zero, [t0,t1], [sim_results[-1][-2,coord], sim_results[-1][-1,coord]] ) sim_tvecs[-1][-1] = t_zero #newIC = sim_results[-1][minidx-1,:] #sim_results[-1] = sim_results[-1][:minidx,:] #sim_tvecs[-1] = sim_tvecs[-1][:minidx] #tvec_0 = linspace(tvec_0[0],tvec_0[1],100) #sim_results.append ( odeint(SLIP_ode, newIC, tvec_0, # args=(params,),rtol=1e-9)) #sim_tvecs.append(tvec_0) # concatenate lists sim_data = vstack( [x[:-1,:] for x in sim_results[:-1] if x.shape[0] > 1] + [sim_results[-1],]) sim_time = [sim_tvecs[0],] for idx in arange(1,len(sim_tvecs)): sim_time.append(sim_tvecs[idx] + sim_time[-1][-1]) sim_time = hstack([x[:-1] for x in sim_time[:-1]] + [sim_time[-1],]) return sim_data, sim_time # Section 1: time to touchdown # TODO: make sampling frequency regular t_flight1 = sqrt(-2.*(y0 - y_land)/g) #t_flight = sqrt() tvec_flight1 = .01*arange(t_flight1*100.) vy_flight1 = tvec_flight1*g y_flight1 = y0 + .5*g*(tvec_flight1**2) x_flight1 = vx0*tvec_flight1 vx_flight1 = vx0*ones_like(tvec_flight1) # Section 2: time to vy = 0 # approach: calculate forward -> estimate interval of # zero position of vy -> refine simulation in that interval # until a point with vy sufficiently close to zero is in the # resulting vector params = {'L0' : L0, 'xF' : t_flight1*vx0 + L0*cos(alpha), 'k' : k, 'm' : m, 'g' : g} IC = array([t_flight1*vx0, y_land, vx0, t_flight1*g]) # initial guess: L0*cos(alpha)/vx0 #t_sim1 = L0*cos(alpha)/vx0 # TODO: implement sim_fail check! sim_fail = False try: sim_phase2, t_phase2 = sim_until(IC,params,lambda x: x[3]) t_phase2 += t_flight1 except SimFailError: print 'simulation aborted (phase 2)\n' sim_fail = True # Phase 3: if not sim_fail: L = sqrt(sim_phase2[-1,1]**2 + (sim_phase2[-1,0]-params['xF'])**2 ) dk, dL = dk_dL(L0,k,L,dE) params2 = deepcopy(params) params2['k'] += dk params2['L0'] += dL IC = sim_phase2[-1,:] compression = (lambda x: sqrt( (x[0]-params2['xF'])**2 + x[1]**2) - params2['L0'] ) #print ('L:', L, 'dk', dk, 'dL', dL, 'dE', dE, '\ncompression:', compression(IC), # 'IC', IC) try: sim_phase3, t_phase3 = sim_until(IC, params2,compression) sim_phase3 = sim_phase3[1:,:] t_phase3 = t_phase3[1:] + t_phase2[-1] except SimFailError: print 'simulation aborted (phase 3)\n' sim_fail = True # Phase 4: if not sim_fail: # time to apex # TODO: make sampling frequency regular vy_liftoff = sim_phase3[-1,3] t_flight2 = -1.*vy_liftoff/g #t_flight = sqrt() tvec_flight2 = arange(t_flight2,0,-.001)[::-1] vy_flight2 = tvec_flight2*g + vy_liftoff y_flight2 = (sim_phase3[-1,1] + vy_liftoff*tvec_flight2 + .5*g*(tvec_flight2**2) ) x_flight2 = sim_phase3[-1,0] + sim_phase3[-1,2]*tvec_flight2 vx_flight2 = sim_phase3[-1,2]*ones_like(tvec_flight2) tvec_flight2 += t_phase3[-1] # todo: return data until error if sim_fail: return { 't': None, 'x': None, 'y': None, 'vx': None, 'vy': None, 'sim_fail': sim_fail, 'dk': None, 'dL': None } # finally: concatenate phases x_final = hstack([x_flight1, sim_phase2[:,0], sim_phase3[:,0], x_flight2 ]) y_final = hstack([y_flight1, sim_phase2[:,1], sim_phase3[:,1], y_flight2 ]) vx_final= hstack([vx_flight1, sim_phase2[:,2], sim_phase3[:,2], vx_flight2]) vy_final= hstack([vy_flight1, sim_phase2[:,3], sim_phase3[:,3], vy_flight2]) tvec_final = hstack([tvec_flight1, t_phase2, t_phase3, tvec_flight2 ]) return {'t': tvec_final, 'x': x_final, 'y': y_final, 'vx': vx_final, 'vy': vy_final, 'sim_fail': sim_fail, 'dk': dk, 'dL': dL, #'sim_res':sim_res, #'sim_phase2': sim_phase2_cut, #'t_phase2': t_phase2_cut }
#segf1 = Segment(0.0, 1.0, lambda x: exp(-1/x)) #segf2 = Segment(0.0, 0.5, lambda x:-1/log(x)) #figure() #print estimateDegreeOfZero(lambda x: x**0.5, 0) #n=7.0 #estimateDegreeOfZero(lambda x:(n+1)/(n) * x ** (1/n), 0) #estimateDegreeOfZero(lambda x:pi/2 * sqrt(1 - (x-1) ** 2), 0) #print estimateDegreeOfZero(lambda x: exp(-1/x), 0) # estimateDegreeOfZero(lambda x: 1/(x+x**4), Inf) # estimateDegreeOfZero(lambda x: exp(-x), Inf) #print findinv(lambda x: 1/(1+exp(-x)), a=-1e300, b=1e300, c=0.5, rtol =1e-16, maxiter = 10000) from numpy import ones_like, zeros_like def _pole_test(f, x, pos = True, deriv = False): return str(testPole(f, x, pos, deriv = deriv)) + " " + str(estimateDegreeOfPole(f, x, pos)) + " " + str(estimateDegreeOfPole(f, x, pos, deriv = True)) print("0,", _pole_test(lambda x: ones_like(x), 0)) print("0',", _pole_test(lambda x: ones_like(x), 0, deriv = True)) print("1,", _pole_test(lambda x: zeros_like(x), 0)) print("x,", _pole_test(lambda x: x, 0)) print("x',", _pole_test(lambda x: x, 0, deriv = True)) print("x**1.5,", _pole_test(lambda x: x**1.5, 0)) print("x**0.5,", _pole_test(lambda x: x**0.5, 0)) print("-log(x),", _pole_test(lambda x: -log(x), 0)) print("-log(sqrt(x)),", _pole_test(lambda x: -log(sqrt(x)), 0)) print("-log(-x),", _pole_test(lambda x: -log(-x), 0, pos = False)) print("1+x**0.5,", _pole_test(lambda x: 1+x**0.5, 0)) print("(1+x**0.5)',", _pole_test(lambda x: 1+x**0.5, 0, deriv = True)) print("x*log(x),", _pole_test(lambda x: x*log(x), 0)) print(_pole_test(lambda x: 1+(1*x+7)*x**-2.5, 0)) print(testPole(lambda x: 1.0/abs(2*x-1), 0.5, pos= False)) print(testPole(lambda x: 9.0*abs(2*x-1), 0.5, pos= True))
print "diff(sin(2*x), x, 3) =", diff(sin(2 * x), x, 3) print '''2.10.3.3''' print "series(cos(x), x) =", series(cos(x), x) print "series(1/cos(x), x) =", series(1 / cos(x), x) # Series 설명을 위하여 그래프를 그림 # 그래프 그리기 관련 기능 등을 담고 있는 pylab 모듈을 불러 들임 import pylab x_deg = pylab.arange(-90, 90 + 1) x_rad = pylab.deg2rad(x_deg) y_cos = pylab.cos(x_rad) y_series_1 = 1 * pylab.ones_like(x_rad) y_series_2 = 1 - x_rad ** 2 / 2 y_series_3 = 1 - x_rad ** 2 / 2 + x_rad ** 4 / 24 pylab.plot(x_deg, y_cos, label='cos') pylab.plot(x_deg, y_series_1, label='series 1') pylab.plot(x_deg, y_series_2, label='series 2') pylab.plot(x_deg, y_series_3, label='series 3') pylab.grid() pylab.legend(loc=0) # pylab.show() print '''2.10.3.5''' print "integrate(6*x**5, x) =", integrate(6 * x ** 5, x) print "integrate(sin(x), x) =", integrate(sin(x), x)
def age_specific_rate(model, data_type, reference_area='all', reference_sex='total', reference_year='all', mu_age=None, mu_age_parent=None, sigma_age_parent=None, rate_type='neg_binom', lower_bound=None, interpolation_method='linear', include_covariates=True, zero_re=False): # TODO: expose (and document) interface for alternative rate_type as well as other options, # record reference values in the model """ Generate PyMC objects for model of epidemological age-interval data :Parameters: - `model` : data.ModelData - `data_type` : str, one of 'i', 'r', 'f', 'p', or 'pf' - `reference_area, reference_sex, reference_year` : the node of the model to fit consistently - `mu_age` : pymc.Node, will be used as the age pattern, set to None if not needed - `mu_age_parent` : pymc.Node, will be used as the age pattern of the parent of the root area, set to None if not needed - `sigma_age_parent` : pymc.Node, will be used as the standard deviation of the age pattern, set to None if not needed - `rate_type` : str, optional. One of 'beta_binom', 'binom', 'log_normal_model', 'neg_binom', 'neg_binom_lower_bound_model', 'neg_binom_model', 'normal_model', 'offest_log_normal', or 'poisson' - `lower_bound` : - `interpolation_method` : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, or 'cubic' - `include_covariates` : boolean - `zero_re` : boolean, change one stoch from each set of siblings in area hierarchy to a 'sum to zero' deterministic :Results: - Returns dict of PyMC objects, including 'pi', the covariate adjusted predicted values for each row of data """ name = data_type import data result = data.ModelVars() if (mu_age_parent != None and pl.any(pl.isnan(mu_age_parent))) \ or (sigma_age_parent != None and pl.any(pl.isnan(sigma_age_parent))): mu_age_parent = None sigma_age_parent = None print 'WARNING: nan found in parent mu/sigma. Ignoring' ages = pl.array(model.parameters['ages']) data = model.get_data(data_type) if lower_bound: lb_data = model.get_data(lower_bound) parameters = model.parameters.get(data_type, {}) area_hierarchy = model.hierarchy vars = dismod3.data.ModelVars() vars += dict(data=data) if 'parameter_age_mesh' in parameters: knots = pl.array(parameters['parameter_age_mesh']) else: knots = pl.arange(ages[0], ages[-1] + 1, 5) smoothing_dict = { 'No Prior': pl.inf, 'Slightly': .5, 'Moderately': .05, 'Very': .005 } if 'smoothness' in parameters: smoothing = smoothing_dict[parameters['smoothness']['amount']] else: smoothing = 0. if mu_age == None: vars.update( age_pattern.age_pattern(name, ages=ages, knots=knots, smoothing=smoothing, interpolation_method=interpolation_method)) else: vars.update(dict(mu_age=mu_age, ages=ages)) vars.update( expert_prior_model.level_constraints(name, parameters, vars['mu_age'], ages)) vars.update( expert_prior_model.derivative_constraints(name, parameters, vars['mu_age'], ages)) if mu_age_parent != None: # setup a hierarchical prior on the simliarity between the # consistent estimate here and (inconsistent) estimate for its # parent in the areas hierarchy #weight_dict = {'Unusable': 10., 'Slightly': 10., 'Moderately': 1., 'Very': .1} #weight = weight_dict[parameters['heterogeneity']] vars.update( similarity_prior_model.similar('parent_similarity_%s' % name, vars['mu_age'], mu_age_parent, sigma_age_parent, 0.)) # also use this as the initial value for the age pattern, if it is not already specified if mu_age == None: if isinstance(mu_age_parent, mc.Node): # TODO: test this code initial_mu = mu_age_parent.value else: initial_mu = mu_age_parent for i, k_i in enumerate(knots): vars['gamma'][i].value = (pl.log( initial_mu[k_i - ages[0]])).clip(-12, 6) age_weights = pl.ones_like( vars['mu_age'].value ) # TODO: use age pattern appropriate to the rate type if len(data) > 0: vars.update( age_integrating_model.age_standardize_approx( name, age_weights, vars['mu_age'], data['age_start'], data['age_end'], ages)) # uncomment the following to effectively remove alleffects #if 'random_effects' in parameters: # for i in range(5): # effect = 'sigma_alpha_%s_%d' % (name, i) # parameters['random_effects'][effect] = dict(dist='TruncatedNormal', mu=.0001, sigma=.00001, lower=.00009, upper=.00011) #if 'fixed_effects' in parameters: # for effect in ['x_sex', 'x_LDI_id_Updated_7July2011']: # parameters['fixed_effects'][effect] = dict(dist='normal', mu=.0001, sigma=.00001) if include_covariates: vars.update( covariate_model.mean_covariate_model(name, vars['mu_interval'], data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re)) else: vars.update({'pi': vars['mu_interval']}) ## ensure that all data has uncertainty quantified appropriately # first replace all missing se from ci missing_se = pl.isnan( data['standard_error']) | (data['standard_error'] < 0) data['standard_error'][missing_se] = (data['upper_ci'][missing_se] - data['lower_ci'][missing_se]) / ( 2 * 1.96) # then replace all missing ess with se missing_ess = pl.isnan(data['effective_sample_size']) data['effective_sample_size'][missing_ess] = data['value'][ missing_ess] * (1 - data['value'][missing_ess] ) / data['standard_error'][missing_ess]**2 if rate_type == 'neg_binom': # warn and drop data that doesn't have effective sample size quantified, or is is non-positive missing_ess = pl.isnan(data['effective_sample_size']) | ( data['effective_sample_size'] < 0) if sum(missing_ess) > 0: print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % ( sum(missing_ess), name) data['effective_sample_size'][missing_ess] = 0.0 # warn and change data where ess is unreasonably huge large_ess = data['effective_sample_size'] >= 1.e10 if sum(large_ess) > 0: print 'WARNING: %d rows of %s data have effective sample size exceeding 10 billion.' % ( sum(large_ess), name) data['effective_sample_size'][large_ess] = 1.e10 if 'heterogeneity' in parameters: lower_dict = {'Slightly': 9., 'Moderately': 3., 'Very': 1.} lower = lower_dict[parameters['heterogeneity']] else: lower = 1. # special case, treat pf data as poisson if data_type == 'pf': lower = 1.e12 vars.update( covariate_model.dispersion_covariate_model( name, data, lower, lower * 9.)) vars.update( rate_model.neg_binom_model(name, vars['pi'], vars['delta'], data['value'], data['effective_sample_size'])) elif rate_type == 'log_normal': # warn and drop data that doesn't have effective sample size quantified missing = pl.isnan( data['standard_error']) | (data['standard_error'] < 0) if sum(missing) > 0: print 'WARNING: %d rows of %s data has no quantification of uncertainty.' % ( sum(missing), name) data['standard_error'][missing] = 1.e6 # TODO: allow options for alternative priors for sigma vars['sigma'] = mc.Uniform('sigma_%s' % name, lower=.0001, upper=1., value=.01) #vars['sigma'] = mc.Exponential('sigma_%s'%name, beta=100., value=.01) vars.update( rate_model.log_normal_model(name, vars['pi'], vars['sigma'], data['value'], data['standard_error'])) elif rate_type == 'normal': # warn and drop data that doesn't have standard error quantified missing = pl.isnan( data['standard_error']) | (data['standard_error'] < 0) if sum(missing) > 0: print 'WARNING: %d rows of %s data has no quantification of uncertainty.' % ( sum(missing), name) data['standard_error'][missing] = 1.e6 vars['sigma'] = mc.Uniform('sigma_%s' % name, lower=.0001, upper=.1, value=.01) vars.update( rate_model.normal_model(name, vars['pi'], vars['sigma'], data['value'], data['standard_error'])) elif rate_type == 'binom': missing_ess = pl.isnan(data['effective_sample_size']) | ( data['effective_sample_size'] < 0) if sum(missing_ess) > 0: print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % ( sum(missing_ess), name) data['effective_sample_size'][missing_ess] = 0.0 vars += rate_model.binom(name, vars['pi'], data['value'], data['effective_sample_size']) elif rate_type == 'beta_binom': vars += rate_model.beta_binom(name, vars['pi'], data['value'], data['effective_sample_size']) elif rate_type == 'poisson': missing_ess = pl.isnan(data['effective_sample_size']) | ( data['effective_sample_size'] < 0) if sum(missing_ess) > 0: print 'WARNING: %d rows of %s data has invalid quantification of uncertainty.' % ( sum(missing_ess), name) data['effective_sample_size'][missing_ess] = 0.0 vars += rate_model.poisson(name, vars['pi'], data['value'], data['effective_sample_size']) elif rate_type == 'offset_log_normal': vars['sigma'] = mc.Uniform('sigma_%s' % name, lower=.0001, upper=10., value=.01) vars += rate_model.offset_log_normal(name, vars['pi'], vars['sigma'], data['value'], data['standard_error']) else: raise Exception, 'rate_model "%s" not implemented' % rate_type else: if include_covariates: vars.update( covariate_model.mean_covariate_model(name, [], data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re)) if include_covariates: vars.update( expert_prior_model.covariate_level_constraints( name, model, vars, ages)) if lower_bound and len(lb_data) > 0: vars['lb'] = age_integrating_model.age_standardize_approx( 'lb_%s' % name, age_weights, vars['mu_age'], lb_data['age_start'], lb_data['age_end'], ages) if include_covariates: vars['lb'].update( covariate_model.mean_covariate_model('lb_%s' % name, vars['lb']['mu_interval'], lb_data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re)) else: vars['lb'].update({'pi': vars['lb']['mu_interval']}) vars['lb'].update( covariate_model.dispersion_covariate_model( 'lb_%s' % name, lb_data, 1e12, 1e13) # treat like poisson ) ## ensure that all data has uncertainty quantified appropriately # first replace all missing se from ci missing_se = pl.isnan( lb_data['standard_error']) | (lb_data['standard_error'] <= 0) lb_data['standard_error'][missing_se] = ( lb_data['upper_ci'][missing_se] - lb_data['lower_ci'][missing_se]) / (2 * 1.96) # then replace all missing ess with se missing_ess = pl.isnan(lb_data['effective_sample_size']) lb_data['effective_sample_size'][missing_ess] = lb_data['value'][ missing_ess] * (1 - lb_data['value'][missing_ess] ) / lb_data['standard_error'][missing_ess]**2 # warn and drop lb_data that doesn't have effective sample size quantified missing_ess = pl.isnan(lb_data['effective_sample_size']) | ( lb_data['effective_sample_size'] <= 0) if sum(missing_ess) > 0: print 'WARNING: %d rows of %s lower bound data has no quantification of uncertainty.' % ( sum(missing_ess), name) lb_data['effective_sample_size'][missing_ess] = 1.0 vars['lb'].update( rate_model.neg_binom_lower_bound_model( 'lb_%s' % name, vars['lb']['pi'], vars['lb']['delta'], lb_data['value'], lb_data['effective_sample_size'])) result[data_type] = vars return result
dm.params['region_effect_prevalence'] = dict(std=level) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([]) for level in [.1, 1, 10.]: dm = initialize_model() dm.params['beta_effect_prevalence'] = dict(mean=[0.], std=[level]) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([]) for level in [.2, 2., 20.]: dm = initialize_model() dm.params['gamma_effect_prevalence'] = dict(mean=list(pl.zeros_like(dm.get_estimate_age_mesh())), std=list(level*pl.ones_like(dm.get_estimate_age_mesh()))) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) # this should change uncertainty, although it turns out not to change levels models.append([]) for level in [.025, .25, 2.5]: dm = initialize_model() dm.params['delta_effect_prevalence'] = dict(mean=3., std=level) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([])
## setup lognormal prior on intercept gamma = mc.Normal('gamma', 0., 2.**-2, value=pl.log(data[:,1].mean())) mu = mc.Lambda('mu', lambda gamma=gamma: pl.exp(gamma)) ## setup Gaussian Process prior on age pattern @mc.deterministic def M(mu=mu): return mc.gp.Mean(lambda x: mu*pl.ones(len(x))) C = mc.gp.FullRankCovariance(mc.gp.matern.euclidean, amp=data[:,1].max(), scale=scale[smoothness], diff_degree=2) sm = mc.gp.GPSubmodel('sm', M, C, in_mesh, init_vals=mu.value*pl.ones_like(in_mesh)) @mc.deterministic def f(f=sm.f_eval): return dismod3.utils.interpolate(in_mesh, f, out_mesh, kind=kind) ## condition on rate being positive @mc.potential def positive(f=f): if pl.any(f < 0.): return -pl.inf else: return 0. ## likelihood of observed data, using normal model for simplicity @mc.deterministic
def Delta(x, tolerance): """Approximates delta function for plotting: Equals one when x is within tolerance of zero.""" y = py.ones_like(x) y[x > tolerance] = 0 y[x <= -tolerance] = 0 return y
# +-----+ # # In equations this represents # $$S' = b-m,$$ # $$b = h_b S,$$ # $$m = h_m S.$$ # # And then if $h_b$ and $h_m$ are constants, the closed form solution is # $$S = S_0 e^{(h_b-h_m)t}.$$ # <codecell> t = 1. * pl.arange(101) S_0 = 1. h_b = .02 * pl.ones_like(t) h_m = .01 * pl.ones_like(t) # <codecell> def one_compartment_ode(S, t, h_b, h_m): # piecewise-constant functions of time implementend as array t = int(pl.clip(t, 0, len(h_b) - 1)) return (h_b[t] - h_m[t]) * S # <markdowncell> # SciPy has two wrappers for the odepack solver, <code>scipy.integrate.odeint</code> (function-based) and <code>scipy.integrate.ode</code> (object-based). #
def validate_ai_re(N=500, delta_true=.15, sigma_true=[.1,.1,.1,.1,.1], pi_true=quadratic, smoothness='Moderately', heterogeneity='Slightly'): ## generate simulated data a = pl.arange(0, 101, 1) pi_age_true = pi_true(a) import dismod3 import simplejson as json model = data.ModelData.from_gbd_jsons(json.loads(dismod3.disease_json.DiseaseJson().to_json())) gbd_hierarchy = model.hierarchy model = data_simulation.simple_model(N) model.hierarchy = gbd_hierarchy model.parameters['p']['parameter_age_mesh'] = range(0, 101, 10) model.parameters['p']['smoothness'] = dict(amount=smoothness) model.parameters['p']['heterogeneity'] = heterogeneity age_start = pl.array(mc.runiform(0, 100, size=N), dtype=int) age_end = pl.array(mc.runiform(age_start, 100, size=N), dtype=int) age_weights = pl.ones_like(a) sum_pi_wt = pl.cumsum(pi_age_true*age_weights) sum_wt = pl.cumsum(age_weights*1.) p = (sum_pi_wt[age_end] - sum_pi_wt[age_start]) / (sum_wt[age_end] - sum_wt[age_start]) # correct cases where age_start == age_end i = age_start == age_end if pl.any(i): p[i] = pi_age_true[age_start[i]] model.input_data['age_start'] = age_start model.input_data['age_end'] = age_end model.input_data['effective_sample_size'] = mc.runiform(100, 10000, size=N) from validate_covariates import alpha_true_sim area_list = pl.array(['all', 'super-region_3', 'north_africa_middle_east', 'EGY', 'KWT', 'IRN', 'IRQ', 'JOR', 'SYR']) alpha = alpha_true_sim(model, area_list, sigma_true) print alpha model.input_data['true'] = pl.nan model.input_data['area'] = area_list[mc.rcategorical(pl.ones(len(area_list)) / float(len(area_list)), N)] for i, a in model.input_data['area'].iteritems(): model.input_data['true'][i] = p[i] * pl.exp(pl.sum([alpha[n] for n in nx.shortest_path(model.hierarchy, 'all', a) if n in alpha])) p = model.input_data['true'] n = model.input_data['effective_sample_size'] model.input_data['value'] = mc.rnegative_binomial(n*p, delta_true*n*p) / n ## Then fit the model and compare the estimates to the truth model.vars = {} model.vars['p'] = data_model.data_model('p', model, 'p', 'north_africa_middle_east', 'total', 'all', None, None, None) #model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'], iter=1005, burn=500, thin=5, tune_interval=100) model.map, model.mcmc = fit_model.fit_data_model(model.vars['p'], iter=10000, burn=5000, thin=25, tune_interval=100) graphics.plot_one_ppc(model.vars['p'], 'p') graphics.plot_convergence_diag(model.vars) graphics.plot_one_type(model, model.vars['p'], {}, 'p') pl.plot(range(101), pi_age_true, 'r:', label='Truth') pl.legend(fancybox=True, shadow=True, loc='upper left') pl.show() model.input_data['mu_pred'] = model.vars['p']['p_pred'].stats()['mean'] model.input_data['sigma_pred'] = model.vars['p']['p_pred'].stats()['standard deviation'] data_simulation.add_quality_metrics(model.input_data) model.delta = pandas.DataFrame(dict(true=[delta_true])) model.delta['mu_pred'] = pl.exp(model.vars['p']['eta'].trace()).mean() model.delta['sigma_pred'] = pl.exp(model.vars['p']['eta'].trace()).std() data_simulation.add_quality_metrics(model.delta) model.alpha = pandas.DataFrame(index=[n for n in nx.traversal.dfs_preorder_nodes(model.hierarchy)]) model.alpha['true'] = pandas.Series(dict(alpha)) model.alpha['mu_pred'] = pandas.Series([n.stats()['mean'] for n in model.vars['p']['alpha']], index=model.vars['p']['U'].columns) model.alpha['sigma_pred'] = pandas.Series([n.stats()['standard deviation'] for n in model.vars['p']['alpha']], index=model.vars['p']['U'].columns) model.alpha = model.alpha.dropna() data_simulation.add_quality_metrics(model.alpha) model.sigma = pandas.DataFrame(dict(true=sigma_true)) model.sigma['mu_pred'] = [n.stats()['mean'] for n in model.vars['p']['sigma_alpha']] model.sigma['sigma_pred']=[n.stats()['standard deviation'] for n in model.vars['p']['sigma_alpha']] data_simulation.add_quality_metrics(model.sigma) print 'delta' print model.delta print '\ndata prediction bias: %.5f, MARE: %.3f, coverage: %.2f' % (model.input_data['abs_err'].mean(), pl.median(pl.absolute(model.input_data['rel_err'].dropna())), model.input_data['covered?'].mean()) model.mu = pandas.DataFrame(dict(true=pi_age_true, mu_pred=model.vars['p']['mu_age'].stats()['mean'], sigma_pred=model.vars['p']['mu_age'].stats()['standard deviation'])) data_simulation.add_quality_metrics(model.mu) data_simulation.initialize_results(model) data_simulation.add_to_results(model, 'delta') data_simulation.add_to_results(model, 'mu') data_simulation.add_to_results(model, 'input_data') data_simulation.add_to_results(model, 'alpha') data_simulation.add_to_results(model, 'sigma') data_simulation.finalize_results(model) print model.results return model
print "diff(tan(x), x) = ", diff(tan(x), x) print "limit((tan(x+y)-tan(x))/y,y,0) =", limit((tan(x + y) - tan(x)) / y, y, 0) print "diff(sin(2*x), x, 1) =", diff(sin(2 * x), x, 1) print "diff(sin(2*x), x, 2) =", diff(sin(2 * x), x, 2) print "diff(sin(2*x), x, 3) =", diff(sin(2 * x), x, 3) print '''2.10.3.3''' print "series(cos(x), x) =", series(cos(x), x) print "series(1/cos(x), x) = ", series(1 / cos(x), x) import pylab x_deg = pylab.arange(-90, 90 + 1) x_rad = pylab.deg2rad(x_deg) y_cos = pylab.cos(x_rad) y_series_1 = 1 * pylab.ones_like(x_rad) y_series_2 = 1 - x_rad**2 / 2 y_series_3 = 1 - x_rad**2 / 2 + x_rad**4 / 24 pylab.plot(x_deg, y_cos, label='cos') pylab.plot(x_deg, y_series_1, label='series 1') pylab.plot(x_deg, y_series_2, label='series 2') pylab.plot(x_deg, y_series_3, label='series 3') pylab.grid() pylab.legend(loc=0) #pylab.show() print '''2.10.3.5''' print "integrate(6*x**5, x) =", integrate(6 * x**5, x) print "integrate(sin(x), x) =", integrate(sin(x), x) print "integrate(log(x), x) =", integrate(log(x), x)
# +-----+ # # In equations this represents # $$S' = b-m,$$ # $$b = h_b S,$$ # $$m = h_m S.$$ # # And then if $h_b$ and $h_m$ are constants, the closed form solution is # $$S = S_0 e^{(h_b-h_m)t}.$$ # <codecell> t = 1.*pl.arange(101) S_0 = 1. h_b = .02*pl.ones_like(t) h_m = .01*pl.ones_like(t) # <codecell> def one_compartment_ode(S, t, h_b, h_m): # piecewise-constant functions of time implementend as array t = int(pl.clip(t, 0, len(h_b)-1)) return (h_b[t]-h_m[t])*S # <markdowncell> # SciPy has two wrappers for the odepack solver, <code>scipy.integrate.odeint</code> (function-based) and <code>scipy.integrate.ode</code> (object-based). # # The function-based wrapper has less overhead, so I'll use it here. The object-based wrapper has a nicer format for some applications.
dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \ [d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \ [d for d in dm.data if d.get('country_iso3_code') == 'GBR'] dm.params['global_priors']['level_bounds']['excess_mortality'] = dict(lower=.2, upper=10.) fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False) models.append(dm) dm = initialize_model() dm.description = 'With excess-mortality data and priors removed' dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \ [d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \ [d for d in dm.data if d.get('country_iso3_code') == 'GBR'] dm.params['global_priors']['level_bounds']['excess_mortality'] = dict(lower=0., upper=10.) dm.params['global_priors']['smoothness']['prevalence']['amount'] = 'No Prior' dm.params['gamma_effect_excess-mortality'] = dict(mean=list(pl.zeros_like(dm.get_estimate_age_mesh())), std=list(10.*pl.ones_like(dm.get_estimate_age_mesh()))) fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False) models.append(dm) dm = initialize_model() dm.description = 'Without increasing prior on excess-mortality, but with "Very" smoothing' dm.data = [d for d in dm.data if dm.relevant_to(d, 'all-cause_mortality', region, year, sex)] + \ [d for d in dm.data if dm.relevant_to(d, 'prevalence_x_excess-mortality', region, year, sex)] + \ [d for d in dm.data if d.get('country_iso3_code') == 'GBR'] + \ [d for d in dm.data if dm.relevant_to(d, 'excess-mortality', 'all', 'all', 'all')] dm.params['global_priors']['smoothness']['prevalence']['amount'] = 'Very' fit_posterior.fit_posterior(dm, region, sex, year, map_only=True, store_results=False) models.append(dm) dm = initialize_model()
def delta(eta=eta): return pl.exp(eta) * pl.ones_like(input_data.index)
dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([]) for level in [.1, 1, 10.]: dm = initialize_model() dm.params['beta_effect_prevalence'] = dict(mean=[0.], std=[level]) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([]) for level in [.2, 2., 20.]: dm = initialize_model() dm.params['gamma_effect_prevalence'] = dict( mean=list(pl.zeros_like(dm.get_estimate_age_mesh())), std=list(level * pl.ones_like(dm.get_estimate_age_mesh()))) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) # this should change uncertainty, although it turns out not to change levels models.append([]) for level in [.025, .25, 2.5]: dm = initialize_model() dm.params['delta_effect_prevalence'] = dict(mean=3., std=level) dismod3.neg_binom_model.fit_emp_prior(dm, 'prevalence', map_only=True) models[-1].append(dm) models.append([]) for level in [1, 2, 3]: dm = initialize_model() dm.params['delta_effect_prevalence'] = dict(mean=level, std=.25)
pl.figure(figsize=(11, 8.5), dpi=120) pl.subplots_adjust(wspace=0.4) pl.subplot(2, 2, 1) plot_funnel(0.004, "5") pl.subplot(2, 2, 2) plot_funnel(0.004, "50") pl.subplot(2, 1, 2) r = pl.array(schiz["r"]) n = pl.array(schiz["n"]) pi = mc.Uniform("pi", 0, 1, value=0.001) delta = mc.Uniform("delta", 0, 100, value=0.0001) nb = rate_model.neg_binom_model("funnel", pi * pl.ones_like(n), delta * pl.ones_like(n), r, n) # old way: # nb = rate_model.neg_binom_model('funnel', pi, delta*r*n, r, n) mcmc = mc.MCMC([pi, delta, nb]) mcmc.sample(20000, 10000, 10, verbose=False, progress_bar=False) sorted_indices = r.argsort().argsort() jitter = mc.rnormal(0, 0.1 ** -2, len(nb["p_pred"].trace())) for i, s_i in enumerate(sorted_indices): pl.plot(s_i + jitter, nb["p_pred"].trace()[:, i], "ko", mew=0, alpha=0.25, zorder=-99) pl.errorbar( sorted_indices, r, yerr=1.96 * pl.sqrt(r * (1 - r) / n), fmt="ws", mew=1, mec="white", ms=5, elinewidth=3, capsize=0 ) pl.errorbar(sorted_indices, r, yerr=1.96 * pl.sqrt(r * (1 - r) / n), fmt="ks", mew=1, mec="white", ms=5)
def set_birth_prev(value): model.vars['logit_C0'].value = mc.logit(pl.maximum(1.e-9, value)) ### @export 'congenital' set_rate('remission', pl.zeros_like(ages)) set_rate('incidence', pl.zeros_like(ages)) set_birth_prev(.2) set_rate('excess-mortality', .5*(ages/100.)**2) book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], xticks=[0,25,50,75,100], panel='b') pl.savefig('book/graphics/forward-sim-congenital.pdf') ### @export 'mental' set_rate('remission', pl.ones_like(ages)*.25) set_rate('incidence', pl.where((ages > 14) & (ages < 100), (100-ages)*.001, 0.)) set_birth_prev(0) set_rate('excess-mortality', 1e-4*pl.ones_like(ages)) book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], panel='a') pl.savefig('book/graphics/forward-sim-mental.pdf') ### @export 'old_age' set_rate('remission', pl.ones_like(ages)*0.) set_rate('incidence', pl.where(ages > 50, pl.exp((ages-50.)/25.)*.01, 0.)) set_birth_prev(0) set_rate('excess-mortality', pl.exp(ages/25.)*.01) book_graphics.plot_age_patterns(model, yticks=[0,.2,.4], panel='c')
def age_specific_rate( model, data_type, reference_area="all", reference_sex="total", reference_year="all", mu_age=None, mu_age_parent=None, sigma_age_parent=None, rate_type="neg_binom", lower_bound=None, interpolation_method="linear", include_covariates=True, zero_re=False, ): # TODO: expose (and document) interface for alternative rate_type as well as other options, # record reference values in the model """ Generate PyMC objects for model of epidemological age-interval data :Parameters: - `model` : data.ModelData - `data_type` : str, one of 'i', 'r', 'f', 'p', or 'pf' - `reference_area, reference_sex, reference_year` : the node of the model to fit consistently - `mu_age` : pymc.Node, will be used as the age pattern, set to None if not needed - `mu_age_parent` : pymc.Node, will be used as the age pattern of the parent of the root area, set to None if not needed - `sigma_age_parent` : pymc.Node, will be used as the standard deviation of the age pattern, set to None if not needed - `rate_type` : str, optional. One of 'beta_binom', 'binom', 'log_normal_model', 'neg_binom', 'neg_binom_lower_bound_model', 'neg_binom_model', 'normal_model', 'offest_log_normal', or 'poisson' - `lower_bound` : - `interpolation_method` : str, optional, one of 'linear', 'nearest', 'zero', 'slinear', 'quadratic, or 'cubic' - `include_covariates` : boolean - `zero_re` : boolean, change one stoch from each set of siblings in area hierarchy to a 'sum to zero' deterministic :Results: - Returns dict of PyMC objects, including 'pi', the covariate adjusted predicted values for each row of data """ name = data_type import data result = data.ModelVars() if (mu_age_parent != None and pl.any(pl.isnan(mu_age_parent))) or ( sigma_age_parent != None and pl.any(pl.isnan(sigma_age_parent)) ): mu_age_parent = None sigma_age_parent = None print "WARNING: nan found in parent mu/sigma. Ignoring" ages = pl.array(model.parameters["ages"]) data = model.get_data(data_type) if lower_bound: lb_data = model.get_data(lower_bound) parameters = model.parameters.get(data_type, {}) area_hierarchy = model.hierarchy vars = dismod3.data.ModelVars() vars += dict(data=data) if "parameter_age_mesh" in parameters: knots = pl.array(parameters["parameter_age_mesh"]) else: knots = pl.arange(ages[0], ages[-1] + 1, 5) smoothing_dict = {"No Prior": pl.inf, "Slightly": 0.5, "Moderately": 0.05, "Very": 0.005} if "smoothness" in parameters: smoothing = smoothing_dict[parameters["smoothness"]["amount"]] else: smoothing = 0.0 if mu_age == None: vars.update( age_pattern.age_pattern( name, ages=ages, knots=knots, smoothing=smoothing, interpolation_method=interpolation_method ) ) else: vars.update(dict(mu_age=mu_age, ages=ages)) vars.update(expert_prior_model.level_constraints(name, parameters, vars["mu_age"], ages)) vars.update(expert_prior_model.derivative_constraints(name, parameters, vars["mu_age"], ages)) if mu_age_parent != None: # setup a hierarchical prior on the simliarity between the # consistent estimate here and (inconsistent) estimate for its # parent in the areas hierarchy # weight_dict = {'Unusable': 10., 'Slightly': 10., 'Moderately': 1., 'Very': .1} # weight = weight_dict[parameters['heterogeneity']] vars.update( similarity_prior_model.similar( "parent_similarity_%s" % name, vars["mu_age"], mu_age_parent, sigma_age_parent, 0.0 ) ) # also use this as the initial value for the age pattern, if it is not already specified if mu_age == None: if isinstance(mu_age_parent, mc.Node): # TODO: test this code initial_mu = mu_age_parent.value else: initial_mu = mu_age_parent for i, k_i in enumerate(knots): vars["gamma"][i].value = (pl.log(initial_mu[k_i - ages[0]])).clip(-12, 6) age_weights = pl.ones_like(vars["mu_age"].value) # TODO: use age pattern appropriate to the rate type if len(data) > 0: vars.update( age_integrating_model.age_standardize_approx( name, age_weights, vars["mu_age"], data["age_start"], data["age_end"], ages ) ) # uncomment the following to effectively remove alleffects # if 'random_effects' in parameters: # for i in range(5): # effect = 'sigma_alpha_%s_%d' % (name, i) # parameters['random_effects'][effect] = dict(dist='TruncatedNormal', mu=.0001, sigma=.00001, lower=.00009, upper=.00011) # if 'fixed_effects' in parameters: # for effect in ['x_sex', 'x_LDI_id_Updated_7July2011']: # parameters['fixed_effects'][effect] = dict(dist='normal', mu=.0001, sigma=.00001) if include_covariates: vars.update( covariate_model.mean_covariate_model( name, vars["mu_interval"], data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re, ) ) else: vars.update({"pi": vars["mu_interval"]}) ## ensure that all data has uncertainty quantified appropriately # first replace all missing se from ci missing_se = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0) data["standard_error"][missing_se] = (data["upper_ci"][missing_se] - data["lower_ci"][missing_se]) / (2 * 1.96) # then replace all missing ess with se missing_ess = pl.isnan(data["effective_sample_size"]) data["effective_sample_size"][missing_ess] = ( data["value"][missing_ess] * (1 - data["value"][missing_ess]) / data["standard_error"][missing_ess] ** 2 ) if rate_type == "neg_binom": # warn and drop data that doesn't have effective sample size quantified, or is is non-positive missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0) if sum(missing_ess) > 0: print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % ( sum(missing_ess), name, ) data["effective_sample_size"][missing_ess] = 0.0 # warn and change data where ess is unreasonably huge large_ess = data["effective_sample_size"] >= 1.0e10 if sum(large_ess) > 0: print "WARNING: %d rows of %s data have effective sample size exceeding 10 billion." % ( sum(large_ess), name, ) data["effective_sample_size"][large_ess] = 1.0e10 if "heterogeneity" in parameters: lower_dict = {"Slightly": 9.0, "Moderately": 3.0, "Very": 1.0} lower = lower_dict[parameters["heterogeneity"]] else: lower = 1.0 # special case, treat pf data as poisson if data_type == "pf": lower = 1.0e12 vars.update(covariate_model.dispersion_covariate_model(name, data, lower, lower * 9.0)) vars.update( rate_model.neg_binom_model( name, vars["pi"], vars["delta"], data["value"], data["effective_sample_size"] ) ) elif rate_type == "log_normal": # warn and drop data that doesn't have effective sample size quantified missing = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0) if sum(missing) > 0: print "WARNING: %d rows of %s data has no quantification of uncertainty." % (sum(missing), name) data["standard_error"][missing] = 1.0e6 # TODO: allow options for alternative priors for sigma vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=1.0, value=0.01) # vars['sigma'] = mc.Exponential('sigma_%s'%name, beta=100., value=.01) vars.update( rate_model.log_normal_model(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"]) ) elif rate_type == "normal": # warn and drop data that doesn't have standard error quantified missing = pl.isnan(data["standard_error"]) | (data["standard_error"] < 0) if sum(missing) > 0: print "WARNING: %d rows of %s data has no quantification of uncertainty." % (sum(missing), name) data["standard_error"][missing] = 1.0e6 vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=0.1, value=0.01) vars.update(rate_model.normal_model(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"])) elif rate_type == "binom": missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0) if sum(missing_ess) > 0: print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % ( sum(missing_ess), name, ) data["effective_sample_size"][missing_ess] = 0.0 vars += rate_model.binom(name, vars["pi"], data["value"], data["effective_sample_size"]) elif rate_type == "beta_binom": vars += rate_model.beta_binom(name, vars["pi"], data["value"], data["effective_sample_size"]) elif rate_type == "poisson": missing_ess = pl.isnan(data["effective_sample_size"]) | (data["effective_sample_size"] < 0) if sum(missing_ess) > 0: print "WARNING: %d rows of %s data has invalid quantification of uncertainty." % ( sum(missing_ess), name, ) data["effective_sample_size"][missing_ess] = 0.0 vars += rate_model.poisson(name, vars["pi"], data["value"], data["effective_sample_size"]) elif rate_type == "offset_log_normal": vars["sigma"] = mc.Uniform("sigma_%s" % name, lower=0.0001, upper=10.0, value=0.01) vars += rate_model.offset_log_normal(name, vars["pi"], vars["sigma"], data["value"], data["standard_error"]) else: raise Exception, 'rate_model "%s" not implemented' % rate_type else: if include_covariates: vars.update( covariate_model.mean_covariate_model( name, [], data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re ) ) if include_covariates: vars.update(expert_prior_model.covariate_level_constraints(name, model, vars, ages)) if lower_bound and len(lb_data) > 0: vars["lb"] = age_integrating_model.age_standardize_approx( "lb_%s" % name, age_weights, vars["mu_age"], lb_data["age_start"], lb_data["age_end"], ages ) if include_covariates: vars["lb"].update( covariate_model.mean_covariate_model( "lb_%s" % name, vars["lb"]["mu_interval"], lb_data, parameters, model, reference_area, reference_sex, reference_year, zero_re=zero_re, ) ) else: vars["lb"].update({"pi": vars["lb"]["mu_interval"]}) vars["lb"].update( covariate_model.dispersion_covariate_model("lb_%s" % name, lb_data, 1e12, 1e13) # treat like poisson ) ## ensure that all data has uncertainty quantified appropriately # first replace all missing se from ci missing_se = pl.isnan(lb_data["standard_error"]) | (lb_data["standard_error"] <= 0) lb_data["standard_error"][missing_se] = (lb_data["upper_ci"][missing_se] - lb_data["lower_ci"][missing_se]) / ( 2 * 1.96 ) # then replace all missing ess with se missing_ess = pl.isnan(lb_data["effective_sample_size"]) lb_data["effective_sample_size"][missing_ess] = ( lb_data["value"][missing_ess] * (1 - lb_data["value"][missing_ess]) / lb_data["standard_error"][missing_ess] ** 2 ) # warn and drop lb_data that doesn't have effective sample size quantified missing_ess = pl.isnan(lb_data["effective_sample_size"]) | (lb_data["effective_sample_size"] <= 0) if sum(missing_ess) > 0: print "WARNING: %d rows of %s lower bound data has no quantification of uncertainty." % ( sum(missing_ess), name, ) lb_data["effective_sample_size"][missing_ess] = 1.0 vars["lb"].update( rate_model.neg_binom_lower_bound_model( "lb_%s" % name, vars["lb"]["pi"], vars["lb"]["delta"], lb_data["value"], lb_data["effective_sample_size"], ) ) result[data_type] = vars return result