def powerVsThetaInAnnulus(self,lLower,lUpper,deltaTheta=2.0,powerOfL=0,\
fitSpline=False,show=False,cutByMask=False):
"""
@brief Given an anuulus, radially averages the power spectrum to produce a
function of angle theta \f$P(\theta)\f$
@param lLower Lower bound of the annulus
@param lUpper Upper bound of the annulus
@param deltaTheta Width of bins in theta
@param powerOfL The power of L to multiply to PS with, before radial averaging
@param fitspline If True returns a spline fit to the function
@param show If True displays the function \f$P(\theta)\f$
@param cutByMask If a kMask exists with p2d, then skip over masked pixels
@return (If fitSpline:) binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
@return (else:) binnedPA, binnedTheta, binCount, binStdDev
"""
if not(cutByMask):
a= (self.modLMap < lUpper)
b= (self.modLMap > lLower)
else:
a= ((self.modLMap < lUpper)*(self.kMask>0))
b= ((self.modLMap > lLower)*(self.kMask>0))
c = a*b
indices = numpy.where(c)
p = self.powerMap*self.modLMap**powerOfL
thetaAnnu = numpy.ravel(self.thetaMap[indices])
powerAnnu = (numpy.ravel(p[indices]))
binnedTheta, binnedPA,binStdDev,binCount = utils.bin(thetaAnnu,powerAnnu,deltaTheta)
if fitSpline:
logspl = splrep(binnedTheta,numpy.log(numpy.abs(binnedPA)),s=2,k=4)
median = numpy.median(numpy.abs(binnedPA))
if show:
pylab.plot(binnedTheta,numpy.log(numpy.abs(binnedPA)),'ro')
if fitSpline:
theta = numpy.arange(180.)
logSmPA = splev(theta,logspl)
pylab.plot(theta,logSmPA)
med = theta.copy()*0. +median
pylab.plot(theta,numpy.log(med))
pylab.xlim(-180.,180.)
pylab.xlabel(r'$\theta$',fontsize=15)
pylab.ylabel(r'$\langle \ell^%d C_\ell (\theta)\rangle$'%powerOfL,\
fontsize=15)
pylab.show()
if fitSpline:
logspl = logspl
threshold = median
return binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
else:
return binnedPA, binnedTheta, binCount, binStdDev
def init(mode):
print("检测tunasync是否存在...")
code = 1
cmd = delegator.run("/usr/bin/tunasync -v")
print(cmd.out)
if cmd.return_code != 0:
code = 0
cmd = delegator.run("/usr/bin/tunasynctl -v")
print(cmd.out)
if cmd.return_code != 0:
code = 0
if code == 1:
print("tunasync已存在，无需安装...")
else:
if mode:
mode = int(mode)
else:
mode = 2
print("开始安装...")
try:
if mode == 1:
if bin() < 0:
print(
"安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
)
return 0
else:
pass
else:
if build() < 0:
print(
"安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
)
return 0
else:
pass
except:
print(
"安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
)
return 0
try:
print("开始设置manager...")
init_manager()
print("开始设置服务...")
systemd()
print("启动manager...")
systemd_control('start', 'manager')
print("设置manager自启...")
systemd_control('enable', 'manager')
print("开始设置worker...")
init_worker()
# print("启动worker...")
# systemd_control('start', 'worker')
# print("设置worker自启...")
# systemd_control('enable', 'worker')
print("worker将不会启动直到新增第一个mirror...")
except:
print("设置时出错...")
def powerVsThetaInAnnulus(self,lLower,lUpper,deltaTheta=2.0,powerOfL=0,\
fitSpline=False,show=False,cutByMask=False):
"""
@brief Given an anuulus, radially averages the power spectrum to produce a
function of angle theta \f$P(\theta)\f$
@param lLower Lower bound of the annulus
@param lUpper Upper bound of the annulus
@param deltaTheta Width of bins in theta
@param powerOfL The power of L to multiply to PS with, before radial averaging
@param fitspline If True returns a spline fit to the function
@param show If True displays the function \f$P(\theta)\f$
@param cutByMask If a kMask exists with p2d, then skip over masked pixels
@return (If fitSpline:) binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
@return (else:) binnedPA, binnedTheta, binCount, binStdDev
"""
if not(cutByMask):
a= (self.modLMap < lUpper)
b= (self.modLMap > lLower)
else:
a= ((self.modLMap < lUpper)*(self.kMask>0))
b= ((self.modLMap > lLower)*(self.kMask>0))
c = a*b
indices = np.where(c)
p = self.powerMap*self.modLMap**powerOfL
thetaAnnu = np.ravel(self.thetaMap[indices])
powerAnnu = (np.ravel(p[indices]))
binnedTheta, binnedPA,binStdDev,binCount = utils.bin(thetaAnnu,powerAnnu,deltaTheta)
if fitSpline:
logspl = splrep(binnedTheta,np.log(np.abs(binnedPA)),s=2,k=4)
median = np.median(np.abs(binnedPA))
if show:
pylab.plot(binnedTheta,np.log(np.abs(binnedPA)),'ro')
if fitSpline:
theta = np.arange(180.)
logSmPA = splev(theta,logspl)
pylab.plot(theta,logSmPA)
med = theta.copy()*0. +median
pylab.plot(theta,np.log(med))
pylab.xlim(-180.,180.)
pylab.xlabel(r'$\theta$',fontsize=15)
pylab.ylabel(r'$\langle \ell^%d C_\ell (\theta)\rangle$'%powerOfL,\
fontsize=15)
pylab.show()
if fitSpline:
logspl = logspl
threshold = median
return binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
else:
return binnedPA, binnedTheta, binCount, binStdDev
def evaluate(function, calibrator, dist, n):
zs = dist(size=n)
ps = function(zs)
phats = calibrator.calibrate(zs)
bins = utils.get_discrete_bins(phats)
data = list(zip(phats, ps))
binned_data = utils.bin(data, bins)
return utils.plugin_ce(binned_data)**2
def eval_top_calibration(probs, logits, labels, plugin=True):
correct = (utils.get_top_predictions(logits) == labels)
data = list(zip(probs, correct))
bins = utils.get_discrete_bins(probs)
binned_data = utils.bin(data, bins)
if plugin:
return utils.plugin_ce(binned_data)**2
else:
return utils.improved_unbiased_square_ce(binned_data)
def polar_dot_binned(thetas,
at=1,
color='C0',
spread=0.25,
num_bins=50,
ax=None,
**kwargs):
"""Plots histogrammed/binned points onto the outside of a polar Axes.
Parameters
----------
thetas : 1D array-like
Angular values to plot.
at : int or float (optional, default=1)
Position (r-value, along radial axis) to plot points.
color : valid matplotlib color (optional, default='C0')
Color of dots.
spread : float (optional, default=2.)
How spread out should the dots within the histogram bin be?
num_bins : int (optional, default=50)
How many bins should be used for generating the histogram?
ax : matplotlib Axes handle (optional, default=None)
Axes to plot dots. If `ax` is passed, make sure that it is already
formatted in polar coordinates!
**kwargs : keyword arguments to be passed to ax.scatter()
Returns
-------
fig, ax : matplotlib Figure & Axes handle.
"""
thetas = np.asarray(thetas)
rr, tt = utils.bin(thetas,
at=at,
num_bins=num_bins,
align='left',
spread=spread)
if ax is None:
fig, ax = plt.subplots(subplot_kw=dict(polar=True))
else:
fig = plt.gcf()
ax.scatter(tt, rr, color=color, **kwargs)
return fig, ax
def eval_marginal_calibration(probs, logits, labels, plugin=True):
ces = [] # Compute the calibration error per class, then take the average.
k = logits.shape[1]
labels_one_hot = utils.get_labels_one_hot(np.array(labels), k)
for c in range(k):
probs_c = probs[:, c]
labels_c = labels_one_hot[:, c]
data_c = list(zip(probs_c, labels_c))
bins_c = utils.get_discrete_bins(probs_c)
binned_data_c = utils.bin(data_c, bins_c)
ce_c = utils.plugin_ce(binned_data_c)**2
ces.append(ce_c)
return np.mean(ces)
def get_mse_ce(logits, labels, ce_est):
mses = []
ces = []
logits = np.array(logits)
labels = np.array(labels)
for l in range(k):
cal_logits_l = platt_binners_equal_points[l](platts[l](
logits[:, l]))
data = list(zip(cal_logits_l, labels[:, l]))
bins_l = utils.get_discrete_bins(cal_logits_l)
binned_data = utils.bin(data, bins_l)
# probs = platts[l](logits[:, l])
# for p in [1, 5, 10, 20, 50, 85, 88.5, 92, 94, 96, 98, 100]:
# print(np.percentile(probs, p))
# import time
# time.sleep(100)
# print('lengths')
# print([len(d) for d in binned_data])
ces.append(ce_est(binned_data))
mses.append(
np.mean([(prob - label)**2 for prob, label in data]))
return np.mean(mses), np.mean(ces)
def estimator(data):
binned_data = utils.bin(data, bins)
return utils.plugin_ce(binned_data, power=lp)
n_scrapes = 10
for i in range(n_scrapes):
tweets = twitter.search(q='#monday')
time.sleep(3)
print "we are on iteration %i of %i"%(i+1, n_scrapes)
for key in tweets['statuses']:
mywords.append(key['text'].encode("ascii","ignore"))
for word in (key['text']).split():
wordLength.append(len(word))
# mywords = mywords.encode("ascii", "ignore")
tags = make_tags(get_tag_counts(mywords), maxsize=80)
create_tag_image(tags, 'cloud_large.png', size=(900, 600), fontname='Lobster')
d = makeWordDict(mywords)
# print d
# for key in d:
# print key , d[key]
# wordLength = sorted(wordLength)
a,b, = bin(inputData = wordLength, binWidth = 1)
# # print wordLength
#
#
makePlot(b,a)
#
# a_1, b_1 = bin(inputData=wordLength, binWidth = 5)
# makePlot(b_1,a_1)
def estimator(data):
binned_data = utils.bin(data, bins)
return estimators[i](binned_data)
def compare_estimators(logits,
labels,
platt_data_size,
bin_data_size,
num_bins,
ver_base_size=2000,
ver_size_increment=1000,
num_resamples=100,
save_name='cmp_est'):
# Convert logits to prediction, probs.
predictions = utils.get_top_predictions(logits)
probs = utils.get_top_probs(logits)
correct = (predictions == labels)
# Platt scale on first chunk of data
platt = utils.get_platt_scaler(probs[:platt_data_size],
correct[:platt_data_size])
platt_probs = platt(probs)
estimator_names = ['biased', 'unbiased']
estimators = [
lambda x: utils.plugin_ce(x)**2, utils.improved_unbiased_square_ce
]
bins = utils.get_equal_bins(platt_probs[:platt_data_size + bin_data_size],
num_bins=num_bins)
binner = utils.get_discrete_calibrator(
platt_probs[platt_data_size:platt_data_size + bin_data_size], bins)
verification_probs = binner(platt_probs[platt_data_size + bin_data_size:])
verification_correct = correct[platt_data_size + bin_data_size:]
verification_data = list(zip(verification_probs, verification_correct))
verification_sizes = list(
range(ver_base_size,
len(verification_probs) + 1, ver_size_increment))
# We want to compare the two estimators when varying the number of samples.
# However, a single point of comparison does not tell us much about the estimators.
# So we use resampling - we resample from the test set many times, and run the estimators
# on the resamples. We stores these values. This gives us a sense of the range of values
# the estimator might output.
# So estimates[i][j][k] stores the estimate when using estimator i, with verification_sizes[j]
# samples, in the k-th resampling.
estimates = np.zeros(
(len(estimators), len(verification_sizes), num_resamples))
# We also store the certified estimates. These represent the upper bounds we get using
# each estimator. They are computing using the std-dev of the estimator estimated by
# Bootstrap.
cert_estimates = np.zeros(
(len(estimators), len(verification_sizes), num_resamples))
for ver_idx, verification_size in zip(range(len(verification_sizes)),
verification_sizes):
for k in range(num_resamples):
# Resample
indices = np.random.choice(list(range(len(verification_data))),
size=verification_size,
replace=True)
cur_verification_data = [verification_data[i] for i in indices]
cur_verification_probs = [verification_probs[i] for i in indices]
bins = utils.get_discrete_bins(cur_verification_probs)
# Compute estimates for each estimator.
for i in range(len(estimators)):
def estimator(data):
binned_data = utils.bin(data, bins)
return estimators[i](binned_data)
cur_estimate = estimator(cur_verification_data)
estimates[i][ver_idx][k] = cur_estimate
# cert_resampling_estimates[j].append(
# cur_estimate + utils.bootstrap_std(cur_verification_data, estimator, num_samples=20))
estimates = np.sort(estimates, axis=-1)
lower_bound = int(0.1 * num_resamples)
median = int(0.5 * num_resamples)
upper_bound = int(0.9 * num_resamples)
lower_estimates = estimates[:, :, lower_bound]
upper_estimates = estimates[:, :, upper_bound]
median_estimates = estimates[:, :, median]
# We can also compute the MSEs of the estimator.
bins = utils.get_discrete_bins(verification_probs)
true_calibration = utils.plugin_ce(utils.bin(verification_data, bins))**2
print(true_calibration)
print(np.sqrt(np.mean(estimates[1, -1, :])))
errors = np.abs(estimates - true_calibration)
accumulated_errors = np.mean(errors, axis=-1)
error_bars_90 = 1.645 * np.std(errors, axis=-1) / np.sqrt(num_resamples)
print(accumulated_errors)
plt.errorbar(verification_sizes,
accumulated_errors[0],
yerr=[error_bars_90[0], error_bars_90[0]],
barsabove=True,
color='red',
capsize=4,
label='plugin')
plt.errorbar(verification_sizes,
accumulated_errors[1],
yerr=[error_bars_90[1], error_bars_90[1]],
barsabove=True,
color='blue',
capsize=4,
label='debiased')
plt.ylabel("Mean-Squared-Error")
plt.xlabel("Number of Samples")
plt.legend(loc='upper right')
plt.show()
plt.ylabel("Number of estimates")
plt.xlabel("Absolute deviation from ground truth")
bins = np.linspace(np.min(errors[:, 0, :]), np.max(errors[:, 0, :]), 40)
plt.hist(errors[0][0], bins, alpha=0.5, label='plugin')
plt.hist(errors[1][0], bins, alpha=0.5, label='debiased')
plt.legend(loc='upper right')
plt.gca().yaxis.set_major_formatter(PercentFormatter(xmax=num_resamples))
plt.show()
def f():
s = 0
for x in xrange(1,1000000):
if isPal(str(x)) and isPal(bin(x)): s+=x
return s
def eval_top_calibration(probs, logits, labels):
correct = (utils.get_top_predictions(logits) == labels)
data = list(zip(probs, correct))
bins = utils.get_discrete_bins(probs)
binned_data = utils.bin(data, bins)
return utils.plugin_ce(binned_data)**2
def striphist(
data,
order=None,
markerfacecolors=None,
markeredgecolors=None,
at=None,
num_bins=50,
align='center',
spread=0.25,
ax=None,
**kwargs
):
"""Strip chart with histogrammed point visualization.
Parameters
----------
data : dictionary
Data to plot. Keys should be string, vals should be 1D array-like.
order : list of string or None
Each string should be a valid key in `data`. This is the order to plot
each group along the x-axis. If None, groups will be plotted
alphabetically from left to right.
markerfacecolors, markeredgecolors : list of valid matplotlib colors or None
Marker face and edge colors of each group to plot. Should be ordered as
in `order`. If None, current matplotlib color cycle will be used to
select colors.
at : list of int or float or None
Where to plot each group along the x-axis. Should be ordered as in
`order`.
num_bins : int
How many bins to use for histogramming/binning data.
align : string
How to display histogrammed points. Options are 'center' or 'left'.
spread : float
How spread out should the histogrammed points be?
ax : matplotlib Axes handle or None
Axes handle for plotting. If None, one will be generated for you.
Parameters
----------
**kwargs : keyword arguments to be passed to ax.scatter().
Returns
-------
fig, ax : matplotlib Figure & Axes handle.
"""
if order is None:
order = sorted(data.keys())
if markerfacecolors is None:
markerfacecolors = ['C{}'.format(i % 10) for i in xrange(len(data))]
if markeredgecolors is None:
markeredgecolors = ['C{}'.format(i % 10) for i in xrange(len(data))]
if at is None:
at = range(1, len(data) + 1)
if ax is None:
fig, ax = plt.subplots()
else:
fig = plt.gcf()
for i, group_name in enumerate(order):
xx, yy = utils.bin(data[group_name], at=at[i], num_bins=num_bins,
align=align, spread=spread)
kwargs.update(
{
'c': markerfacecolors[i],
'edgecolors': markeredgecolors[i]
}
)
ax.scatter(xx, yy, **kwargs)
return fig, ax