def build_grid_and_shocks_jitted(): np.random.seed(seed) # Allocate memory and intitialize z_vec = np.empty(ts_length) σ_vec = np.empty(ts_length) z_vec[0] = 0.0 σ_vec[0] = np.sqrt(d / (1 - v)) # Generate state for t in range(ts_length - 1): # Update state z_vec[t + 1] = ρ * z_vec[t] + ϕ_z * σ_vec[t] * randn() σ2 = v * σ_vec[t]**2 + d + ϕ_σ * randn() σ_vec[t + 1] = np.sqrt(max(σ2, 0)) q1, q2 = 0.05, 0.95 # quantiles z_min, z_max = quantile(z_vec, q1), quantile(z_vec, q2) σ_min, σ_max = quantile(σ_vec, q1), quantile(σ_vec, q2) # Build grid along each state axis z_grid = np.linspace(z_min, z_max, z_grid_size) σ_grid = np.linspace(σ_min, σ_max, σ_grid_size) shocks = randn(2, mc_draw_size) return z_grid, σ_grid, shocks
def __rank(self): self.ranking = [] for r in self.rectangles.itervalues(): for d in r.diagonals.itervalues(): self.ranking.append(d) self.ranking.sort(key=lambda x: x.support(), reverse=True) step = 1 percentiles = [utils.quantile(self.ranking, q).support() for q in xrange(0,100+step,step)] self.logger.info("Percentiles: %s" % list(enumerate(percentiles)))
def get_error_estimates(x, sorted=False): """ Compute the quantiles and return the median, -1 sigma value, and +1 sigma value for the array *x*. `x` input ndarray `sorted` boolean flag to sort array, default=False returns a 3-tuple (median, -1 sigma value, and +1 sigma value) """ xs = numpy.array(x) if not sorted: xs.sort() sigfrac = 0.682689 median = quantile(xs, 0.5) lval = quantile(xs, (1-sigfrac)/2.0) hval = quantile(xs, (1+sigfrac)/2.0) return (median, lval, hval)
def get_error_estimates(x, sorted=False): """ Compute the quantiles and return the median, -1 sigma value, and +1 sigma value for the array *x*. `x` input ndarray `sorted` boolean flag to sort array, default=False returns a 3-tuple (median, -1 sigma value, and +1 sigma value) """ xs = numpy.array(x) if not sorted: xs.sort() sigfrac = 0.682689 median = quantile(xs, 0.5) lval = quantile(xs, (1 - sigfrac) / 2.0) hval = quantile(xs, (1 + sigfrac) / 2.0) return (median, lval, hval)
def __rank(self): self.ranking = [] for r in self.rectangles.itervalues(): for d in r.diagonals.itervalues(): self.ranking.append(d) self.ranking.sort(key=lambda x: x.support(), reverse=True) step = 1 percentiles = [ utils.quantile(self.ranking, q).support() for q in xrange(0, 100 + step, step) ] self.logger.info("Percentiles: %s" % list(enumerate(percentiles)))
def get_values_and_intervals(parameters, weights, labels, use_median=False): param_values = np.zeros((len(parameters), 7)) confidence_values = np.zeros((len(parameters), 6)) for idx, param in enumerate(parameters): print( '-> Calculating histogram for {:}.'.format(labels[idx])) if use_median: central_value = quantile(param, [0.5], weights=weights)[0] else: central_value = weighted_mean(param, weights=weights) # bounds returns [[-1sigma, +1sigma],[-2sigma, +2sigma], [-3sigma, +3sigma]] bounds = minimum_credible_intervals(param, central_value, weights, bins=50) param_values[idx, :] = np.concatenate(([central_value], bounds[:,0], bounds[:,1])) confidence_values[idx, :] = central_value + bounds.flatten() return param_values, confidence_values
) else: logging.info(f'softimpute : MAE = {softimpute_scores["MAE"][-1]}') ### Automatic gamma if args.quantile is not None: NAs = 0.1 * torch.randn(mask.shape).double() + mean_impute( X_true, mask) X_ = (1 - mask) * X_true + mask * NAs idx = np.random.choice(len(X_), min(2000, len(X_)), replace=False) X = X_[idx] dists = ((X[:, None] - X)**2).sum(2).flatten() / 2. dists = dists[dists > 0] gamma = quantile(dists, args.quantile, 0).item() * args.quantile_multiplier logging.info( f"epsilon = {gamma} ({100 * args.quantile}th percentile times {args.quantile_multiplier})" ) else: gamma = args.gamma logging.info(f"epsilon = {gamma} (fixed)") data["gamma"].append(gamma) logging.info("Sinkhorn Imputation") sk = SamplesLoss("sinkhorn", p=2,
def price_guide(item, max_cost_quantile=None): """Fetch pricing info for an item""" results = [] if (item['ItemTypeID'] == 'P' and 'stk0' in item['ItemID']) or \ item['ItemTypeID'] == 'S' or \ item['ItemTypeID'] == 'M': # a sticker sheet, a set, or a minifigure color_ids = [0] else: # a normal item color_ids = color.similar_to(item['ColorID']) for c in color_ids: # perform HTTP request parameters = { 'itemType': item['ItemTypeID'], 'itemNo': item['ItemID'], 'itemSeq': 1, 'colorId': c, 'v': 'P', 'priceGroup': 'Y', 'prDec': 2 } url = "http://www.bricklink.com/catalogPG.asp?" + urllib.urlencode(parameters) html = urllib.urlopen(url).read() # parse page page = BS(html) if len(page.find_all(text='Currently Available')) == 0: # not available in this color :( continue else: # newly found inventory new = [] for td in page.find_all('td'): if td.find('a', recursive=False, href=re.compile('/store.asp')) is not None: # find the td element with a link to a store. Its siblings contain # the interesting bits like price and quantity available store_url = td.find('a')['href'] store_id = int(utils.get_params(store_url)['sID']) quantity = int(td.next_sibling.text) cost_per_unit = float(re.findall('[0-9.]+', td.next_sibling.next_sibling.text)[0]) new.append({ 'item_id': item['ItemID'], 'wanted_color_id': item['ColorID'], 'color_id': c, 'store_id': store_id, 'quantity_available': quantity, 'cost_per_unit': cost_per_unit }) # remove items that cost too much if max_cost_quantile is not None and max_cost_quantile < 1.0: observed_prices = [e['quantity_available'] * [e['cost_per_unit']] for e in new] observed_prices = list(sorted(utils.flatten(observed_prices))) if len(observed_prices) > 0: i = utils.quantile(len(observed_prices)-1, max_cost_quantile) max_price = observed_prices[i] new = filter(lambda x: x['cost_per_unit'] <= max_price, new) # add what's left to the considered inventory results.extend(new) if sum(e['quantity_available'] for e in results) >= item['Qty']: # stop early, we've got everything we need return results return results
return z_vec, h_z_vec, h_c_vec, h_d_vec, g_c_vec, g_d_vec return jitted_sim_paths() def build_grid_and_shocks(self, ts_length=10_000_000, seed=1234, q1=0.025, # lower quantile q2=0.925): # upper quantile paths = self.simulate_paths() z_vec, h_z_vec, h_c_vec, h_d_vec, g_c, g_d = paths # Obtain quantiles of state process time series z_min, z_max = quantile(z_vec, q1), quantile(z_vec, q2) h_z_min, h_z_max = quantile(h_z_vec, q1), quantile(h_z_vec, q2) h_c_min, h_c_max = quantile(h_c_vec, q1), quantile(h_c_vec, q2) h_d_min, h_d_max = quantile(h_d_vec, q1), quantile(h_d_vec, q2) # Build grid along each state axis using these quantiles as bounds self.z_grid = np.linspace(z_min, z_max, self.z_grid_size) self.h_z_grid = np.linspace(h_z_min, h_z_max, self.h_z_grid_size) self.h_c_grid = np.linspace(h_c_min, h_c_max, self.h_c_grid_size) self.h_d_grid = np.linspace(h_d_min, h_d_max, self.h_d_grid_size) # Shocks for Monte Carlo integration np.random.seed(1234) self.shocks = randn(3, self.mc_draw_size)
def percentiles(self): percentiles = [utils.quantile(self.ranking, percentile).support() for percentile in xrange(101)] return percentiles
def plot_all_rocs_quant(path, labels, ident, n_pts=101, alpha=0.05, n_runs=100): plt.figure(figsize=(3, 3)) if ident == "sec4": plt.plot([0, 1], [0, 1], color="grey", alpha=0.5) x_vals = np.linspace(0, 1, n_pts) c_1 = list() c_2 = list() c = list() for i in range(0, n_runs): print("run {}".format(i)) basedir = "{}/run_{}/final_analysis/rocs/{}-test".format(path, i, ident) with open(basedir + "-1-fpr.npy", "rb") as f: fpr_1 = np.load(f) with open(basedir + "-1-tpr.npy", "rb") as f: tpr_1 = np.load(f) c_1.append(get_roc_fixed_step(fpr_1[::-1], tpr_1[::-1], x_vals)) with open(basedir + "-2-fpr.npy", "rb") as f: fpr_2 = np.load(f) with open(basedir + "-2-tpr.npy", "rb") as f: tpr_2 = np.load(f) c_2.append(get_roc_fixed_step(fpr_2[::-1], tpr_2[::-1], x_vals)) with open(basedir + "-main-fpr.npy", "rb") as f: fpr = np.load(f) with open(basedir + "-main-tpr.npy", "rb") as f: tpr = np.load(f) c.append(get_roc_fixed_step(fpr[::-1], tpr[::-1], x_vals)) s_1, s_2, s, sa_1, sa_2, sa = labels area1 = plt.fill_between(x_vals, quantile(c_1, (1-alpha/2), axis=0), quantile(c_1, alpha/2, axis=0), color="green", label=sa_1, alpha=0.25) area2 = plt.fill_between(x_vals, quantile(c_2, (1-alpha/2), axis=0), quantile(c_2, alpha/2, axis=0), color="blue", label=sa_2, alpha=0.25) area = plt.fill_between(x_vals, quantile(c, (1-alpha/2), axis=0), quantile(c, alpha/2, axis=0), color="black", label=sa, alpha=0.25) cur1 = plt.plot(x_vals, np.median(c_1, axis=0), label=s_1, color="green", alpha=0.50) cur2 = plt.plot(x_vals, np.median(c_2, axis=0), label=s_2, color="blue", alpha=0.50) cur = plt.plot(x_vals, np.median(c, axis=0), label=s, color="black", alpha=0.50) # plt.legend() plt.xlabel("$FPR$") plt.ylabel("$TPR$") plt.grid() plt.tight_layout() plt.savefig(path + "/roc_curves_quant_" + ident + ".pdf") plt.figure(figsize=(4, 1)) area1 = plt.fill_between([0], [0], [0], color="green", alpha=0.25, label=sa_1) area2 = plt.fill_between([0], [0], [0], color="blue", alpha=0.25, label=sa_2) area = plt.fill_between([0], [0], [0], color="black", alpha=0.25, label=sa) cur1 = plt.Line2D([0], [0], label=s_1, color="green", alpha=0.50) cur2 = plt.Line2D([0], [0], label=s_2, color="blue", alpha=0.50) cur = plt.Line2D([0], [0], label=s, color="black", alpha=0.50) handles = (area1, area2, area, cur1, cur2, cur) labels = (sa_1, sa_2, sa, s_1, s_2, s) plt.gca().axis('off') plt.legend(handles, labels, loc='center', ncol=2) plt.savefig(path + "/legend-" + ident + ".pdf")
def price_guide(item, max_cost_quantile=None): """Fetch pricing info for an item""" results = [] if (item['ItemTypeID'] == 'P' and 'stk0' in item['ItemID']) or \ item['ItemTypeID'] == 'S' or \ item['ItemTypeID'] == 'M': # a sticker sheet, a set, or a minifigure color_ids = [0] else: # a normal item color_ids = color.similar_to(item['ColorID']) for c in color_ids: # perform HTTP request parameters = { 'itemType': item['ItemTypeID'], 'itemNo': item['ItemID'], 'itemSeq': 1, 'colorId': c, 'v': 'P', 'priceGroup': 'Y', 'prDec': 2 } url = "http://www.bricklink.com/catalogPG.asp?" + urllib.urlencode( parameters) html = urllib.urlopen(url).read() # parse page page = BS(html) if len(page.find_all(text='Currently Available')) == 0: # not available in this color :( continue else: # newly found inventory new = [] for td in page.find_all('td'): if td.find('a', recursive=False, href=re.compile('/store.asp')) is not None: # find the td element with a link to a store. Its siblings contain # the interesting bits like price and quantity available store_url = td.find('a')['href'] store_id = int(utils.get_params(store_url)['sID']) quantity = int(td.next_sibling.text) cost_per_unit = float( re.findall('[0-9.]+', td.next_sibling.next_sibling.text)[0]) new.append({ 'item_id': item['ItemID'], 'wanted_color_id': item['ColorID'], 'color_id': c, 'store_id': store_id, 'quantity_available': quantity, 'cost_per_unit': cost_per_unit }) # remove items that cost too much if max_cost_quantile is not None and max_cost_quantile < 1.0: observed_prices = [ e['quantity_available'] * [e['cost_per_unit']] for e in new ] observed_prices = list(sorted(utils.flatten(observed_prices))) if len(observed_prices) > 0: i = utils.quantile( len(observed_prices) - 1, max_cost_quantile) max_price = observed_prices[i] new = filter(lambda x: x['cost_per_unit'] <= max_price, new) # add what's left to the considered inventory results.extend(new) if sum(e['quantity_available'] for e in results) >= item['Qty']: # stop early, we've got everything we need return results return results