def build_grid_and_shocks_jitted():
np.random.seed(seed)
# Allocate memory and intitialize
z_vec = np.empty(ts_length)
σ_vec = np.empty(ts_length)
z_vec[0] = 0.0
σ_vec[0] = np.sqrt(d / (1 - v))
# Generate state
for t in range(ts_length - 1):
# Update state
z_vec[t + 1] = ρ * z_vec[t] + ϕ_z * σ_vec[t] * randn()
σ2 = v * σ_vec[t]**2 + d + ϕ_σ * randn()
σ_vec[t + 1] = np.sqrt(max(σ2, 0))
q1, q2 = 0.05, 0.95 # quantiles
z_min, z_max = quantile(z_vec, q1), quantile(z_vec, q2)
σ_min, σ_max = quantile(σ_vec, q1), quantile(σ_vec, q2)
# Build grid along each state axis
z_grid = np.linspace(z_min, z_max, z_grid_size)
σ_grid = np.linspace(σ_min, σ_max, σ_grid_size)
shocks = randn(2, mc_draw_size)
return z_grid, σ_grid, shocks
def __rank(self):
self.ranking = []
for r in self.rectangles.itervalues():
for d in r.diagonals.itervalues():
self.ranking.append(d)
self.ranking.sort(key=lambda x: x.support(), reverse=True)
step = 1
percentiles = [utils.quantile(self.ranking, q).support() for q in xrange(0,100+step,step)]
self.logger.info("Percentiles: %s" % list(enumerate(percentiles)))
def get_error_estimates(x, sorted=False):
"""
Compute the quantiles and return the median, -1 sigma value, and +1 sigma
value for the array *x*.
`x` input ndarray
`sorted` boolean flag to sort array, default=False
returns a 3-tuple (median, -1 sigma value, and +1 sigma value)
"""
xs = numpy.array(x)
if not sorted:
xs.sort()
sigfrac = 0.682689
median = quantile(xs, 0.5)
lval = quantile(xs, (1-sigfrac)/2.0)
hval = quantile(xs, (1+sigfrac)/2.0)
return (median, lval, hval)
def get_error_estimates(x, sorted=False):
"""
Compute the quantiles and return the median, -1 sigma value, and +1 sigma
value for the array *x*.
`x` input ndarray
`sorted` boolean flag to sort array, default=False
returns a 3-tuple (median, -1 sigma value, and +1 sigma value)
"""
xs = numpy.array(x)
if not sorted:
xs.sort()
sigfrac = 0.682689
median = quantile(xs, 0.5)
lval = quantile(xs, (1 - sigfrac) / 2.0)
hval = quantile(xs, (1 + sigfrac) / 2.0)
return (median, lval, hval)
def __rank(self):
self.ranking = []
for r in self.rectangles.itervalues():
for d in r.diagonals.itervalues():
self.ranking.append(d)
self.ranking.sort(key=lambda x: x.support(), reverse=True)
step = 1
percentiles = [
utils.quantile(self.ranking, q).support()
for q in xrange(0, 100 + step, step)
]
self.logger.info("Percentiles: %s" % list(enumerate(percentiles)))
def get_values_and_intervals(parameters, weights, labels, use_median=False):
param_values = np.zeros((len(parameters), 7))
confidence_values = np.zeros((len(parameters), 6))
for idx, param in enumerate(parameters):
print( '-> Calculating histogram for {:}.'.format(labels[idx]))
if use_median:
central_value = quantile(param, [0.5], weights=weights)[0]
else:
central_value = weighted_mean(param, weights=weights)
# bounds returns [[-1sigma, +1sigma],[-2sigma, +2sigma], [-3sigma, +3sigma]]
bounds = minimum_credible_intervals(param, central_value, weights, bins=50)
param_values[idx, :] = np.concatenate(([central_value], bounds[:,0], bounds[:,1]))
confidence_values[idx, :] = central_value + bounds.flatten()
return param_values, confidence_values
)
else:
logging.info(f'softimpute : MAE = {softimpute_scores["MAE"][-1]}')
### Automatic gamma
if args.quantile is not None:
NAs = 0.1 * torch.randn(mask.shape).double() + mean_impute(
X_true, mask)
X_ = (1 - mask) * X_true + mask * NAs
idx = np.random.choice(len(X_), min(2000, len(X_)), replace=False)
X = X_[idx]
dists = ((X[:, None] - X)**2).sum(2).flatten() / 2.
dists = dists[dists > 0]
gamma = quantile(dists, args.quantile,
0).item() * args.quantile_multiplier
logging.info(
f"epsilon = {gamma} ({100 * args.quantile}th percentile times {args.quantile_multiplier})"
)
else:
gamma = args.gamma
logging.info(f"epsilon = {gamma} (fixed)")
data["gamma"].append(gamma)
logging.info("Sinkhorn Imputation")
sk = SamplesLoss("sinkhorn",
p=2,
def price_guide(item, max_cost_quantile=None):
"""Fetch pricing info for an item"""
results = []
if (item['ItemTypeID'] == 'P' and 'stk0' in item['ItemID']) or \
item['ItemTypeID'] == 'S' or \
item['ItemTypeID'] == 'M':
# a sticker sheet, a set, or a minifigure
color_ids = [0]
else:
# a normal item
color_ids = color.similar_to(item['ColorID'])
for c in color_ids:
# perform HTTP request
parameters = {
'itemType': item['ItemTypeID'],
'itemNo': item['ItemID'],
'itemSeq': 1,
'colorId': c,
'v': 'P',
'priceGroup': 'Y',
'prDec': 2
}
url = "http://www.bricklink.com/catalogPG.asp?" + urllib.urlencode(parameters)
html = urllib.urlopen(url).read()
# parse page
page = BS(html)
if len(page.find_all(text='Currently Available')) == 0:
# not available in this color :(
continue
else:
# newly found inventory
new = []
for td in page.find_all('td'):
if td.find('a', recursive=False, href=re.compile('/store.asp')) is not None:
# find the td element with a link to a store. Its siblings contain
# the interesting bits like price and quantity available
store_url = td.find('a')['href']
store_id = int(utils.get_params(store_url)['sID'])
quantity = int(td.next_sibling.text)
cost_per_unit = float(re.findall('[0-9.]+',
td.next_sibling.next_sibling.text)[0])
new.append({
'item_id': item['ItemID'],
'wanted_color_id': item['ColorID'],
'color_id': c,
'store_id': store_id,
'quantity_available': quantity,
'cost_per_unit': cost_per_unit
})
# remove items that cost too much
if max_cost_quantile is not None and max_cost_quantile < 1.0:
observed_prices = [e['quantity_available'] * [e['cost_per_unit']] for e in new]
observed_prices = list(sorted(utils.flatten(observed_prices)))
if len(observed_prices) > 0:
i = utils.quantile(len(observed_prices)-1, max_cost_quantile)
max_price = observed_prices[i]
new = filter(lambda x: x['cost_per_unit'] <= max_price, new)
# add what's left to the considered inventory
results.extend(new)
if sum(e['quantity_available'] for e in results) >= item['Qty']:
# stop early, we've got everything we need
return results
return results
return z_vec, h_z_vec, h_c_vec, h_d_vec, g_c_vec, g_d_vec
return jitted_sim_paths()
def build_grid_and_shocks(self,
ts_length=10_000_000,
seed=1234,
q1=0.025, # lower quantile
q2=0.925): # upper quantile
paths = self.simulate_paths()
z_vec, h_z_vec, h_c_vec, h_d_vec, g_c, g_d = paths
# Obtain quantiles of state process time series
z_min, z_max = quantile(z_vec, q1), quantile(z_vec, q2)
h_z_min, h_z_max = quantile(h_z_vec, q1), quantile(h_z_vec, q2)
h_c_min, h_c_max = quantile(h_c_vec, q1), quantile(h_c_vec, q2)
h_d_min, h_d_max = quantile(h_d_vec, q1), quantile(h_d_vec, q2)
# Build grid along each state axis using these quantiles as bounds
self.z_grid = np.linspace(z_min, z_max, self.z_grid_size)
self.h_z_grid = np.linspace(h_z_min, h_z_max, self.h_z_grid_size)
self.h_c_grid = np.linspace(h_c_min, h_c_max, self.h_c_grid_size)
self.h_d_grid = np.linspace(h_d_min, h_d_max, self.h_d_grid_size)
# Shocks for Monte Carlo integration
np.random.seed(1234)
self.shocks = randn(3, self.mc_draw_size)
def percentiles(self):
percentiles = [utils.quantile(self.ranking, percentile).support() for percentile in xrange(101)]
return percentiles
def plot_all_rocs_quant(path, labels, ident,
n_pts=101, alpha=0.05, n_runs=100):
plt.figure(figsize=(3, 3))
if ident == "sec4":
plt.plot([0, 1], [0, 1], color="grey", alpha=0.5)
x_vals = np.linspace(0, 1, n_pts)
c_1 = list()
c_2 = list()
c = list()
for i in range(0, n_runs):
print("run {}".format(i))
basedir = "{}/run_{}/final_analysis/rocs/{}-test".format(path, i, ident)
with open(basedir + "-1-fpr.npy", "rb") as f:
fpr_1 = np.load(f)
with open(basedir + "-1-tpr.npy", "rb") as f:
tpr_1 = np.load(f)
c_1.append(get_roc_fixed_step(fpr_1[::-1], tpr_1[::-1], x_vals))
with open(basedir + "-2-fpr.npy", "rb") as f:
fpr_2 = np.load(f)
with open(basedir + "-2-tpr.npy", "rb") as f:
tpr_2 = np.load(f)
c_2.append(get_roc_fixed_step(fpr_2[::-1], tpr_2[::-1], x_vals))
with open(basedir + "-main-fpr.npy", "rb") as f:
fpr = np.load(f)
with open(basedir + "-main-tpr.npy", "rb") as f:
tpr = np.load(f)
c.append(get_roc_fixed_step(fpr[::-1], tpr[::-1], x_vals))
s_1, s_2, s, sa_1, sa_2, sa = labels
area1 = plt.fill_between(x_vals, quantile(c_1, (1-alpha/2), axis=0),
quantile(c_1, alpha/2, axis=0), color="green",
label=sa_1, alpha=0.25)
area2 = plt.fill_between(x_vals, quantile(c_2, (1-alpha/2), axis=0),
quantile(c_2, alpha/2, axis=0), color="blue",
label=sa_2, alpha=0.25)
area = plt.fill_between(x_vals, quantile(c, (1-alpha/2), axis=0),
quantile(c, alpha/2, axis=0), color="black",
label=sa, alpha=0.25)
cur1 = plt.plot(x_vals, np.median(c_1, axis=0),
label=s_1, color="green", alpha=0.50)
cur2 = plt.plot(x_vals, np.median(c_2, axis=0),
label=s_2, color="blue", alpha=0.50)
cur = plt.plot(x_vals, np.median(c, axis=0),
label=s, color="black", alpha=0.50)
# plt.legend()
plt.xlabel("$FPR$")
plt.ylabel("$TPR$")
plt.grid()
plt.tight_layout()
plt.savefig(path + "/roc_curves_quant_" + ident + ".pdf")
plt.figure(figsize=(4, 1))
area1 = plt.fill_between([0], [0], [0],
color="green", alpha=0.25, label=sa_1)
area2 = plt.fill_between([0], [0], [0],
color="blue", alpha=0.25, label=sa_2)
area = plt.fill_between([0], [0], [0], color="black", alpha=0.25, label=sa)
cur1 = plt.Line2D([0], [0], label=s_1, color="green", alpha=0.50)
cur2 = plt.Line2D([0], [0], label=s_2, color="blue", alpha=0.50)
cur = plt.Line2D([0], [0], label=s, color="black", alpha=0.50)
handles = (area1, area2, area, cur1, cur2, cur)
labels = (sa_1, sa_2, sa, s_1, s_2, s)
plt.gca().axis('off')
plt.legend(handles, labels, loc='center', ncol=2)
plt.savefig(path + "/legend-" + ident + ".pdf")
def price_guide(item, max_cost_quantile=None):
"""Fetch pricing info for an item"""
results = []
if (item['ItemTypeID'] == 'P' and 'stk0' in item['ItemID']) or \
item['ItemTypeID'] == 'S' or \
item['ItemTypeID'] == 'M':
# a sticker sheet, a set, or a minifigure
color_ids = [0]
else:
# a normal item
color_ids = color.similar_to(item['ColorID'])
for c in color_ids:
# perform HTTP request
parameters = {
'itemType': item['ItemTypeID'],
'itemNo': item['ItemID'],
'itemSeq': 1,
'colorId': c,
'v': 'P',
'priceGroup': 'Y',
'prDec': 2
}
url = "http://www.bricklink.com/catalogPG.asp?" + urllib.urlencode(
parameters)
html = urllib.urlopen(url).read()
# parse page
page = BS(html)
if len(page.find_all(text='Currently Available')) == 0:
# not available in this color :(
continue
else:
# newly found inventory
new = []
for td in page.find_all('td'):
if td.find('a', recursive=False,
href=re.compile('/store.asp')) is not None:
# find the td element with a link to a store. Its siblings contain
# the interesting bits like price and quantity available
store_url = td.find('a')['href']
store_id = int(utils.get_params(store_url)['sID'])
quantity = int(td.next_sibling.text)
cost_per_unit = float(
re.findall('[0-9.]+',
td.next_sibling.next_sibling.text)[0])
new.append({
'item_id': item['ItemID'],
'wanted_color_id': item['ColorID'],
'color_id': c,
'store_id': store_id,
'quantity_available': quantity,
'cost_per_unit': cost_per_unit
})
# remove items that cost too much
if max_cost_quantile is not None and max_cost_quantile < 1.0:
observed_prices = [
e['quantity_available'] * [e['cost_per_unit']] for e in new
]
observed_prices = list(sorted(utils.flatten(observed_prices)))
if len(observed_prices) > 0:
i = utils.quantile(
len(observed_prices) - 1, max_cost_quantile)
max_price = observed_prices[i]
new = filter(lambda x: x['cost_per_unit'] <= max_price,
new)
# add what's left to the considered inventory
results.extend(new)
if sum(e['quantity_available'] for e in results) >= item['Qty']:
# stop early, we've got everything we need
return results
return results
