Esempio n. 1
0
 def powerVsThetaInAnnulus(self,lLower,lUpper,deltaTheta=2.0,powerOfL=0,\
                           fitSpline=False,show=False,cutByMask=False):
     """
     @brief Given an anuulus, radially averages the power spectrum to produce a
     function of angle theta \f$P(\theta)\f$
     @param lLower Lower bound of the annulus
     @param lUpper Upper bound of the annulus
     @param deltaTheta Width of bins in theta
     @param powerOfL The power of L to multiply to PS with, before radial averaging
     @param fitspline If True returns a spline fit to the function
     @param show If True displays the function \f$P(\theta)\f$ 
     @param cutByMask If a kMask exists with p2d, then skip over masked pixels
     
     @return (If fitSpline:) binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
     
     @return (else:) binnedPA, binnedTheta, binCount, binStdDev 
     """
     if not(cutByMask):
         a= (self.modLMap < lUpper)
         b= (self.modLMap > lLower)
     else:
         a= ((self.modLMap < lUpper)*(self.kMask>0))
         b= ((self.modLMap > lLower)*(self.kMask>0))
     c = a*b
     
     indices = numpy.where(c)
     p = self.powerMap*self.modLMap**powerOfL
     thetaAnnu = numpy.ravel(self.thetaMap[indices])
     powerAnnu = (numpy.ravel(p[indices]))
     
     binnedTheta, binnedPA,binStdDev,binCount = utils.bin(thetaAnnu,powerAnnu,deltaTheta)
     
     if fitSpline:
         logspl = splrep(binnedTheta,numpy.log(numpy.abs(binnedPA)),s=2,k=4)
         median = numpy.median(numpy.abs(binnedPA))
         
     if show:
         pylab.plot(binnedTheta,numpy.log(numpy.abs(binnedPA)),'ro')
         if fitSpline:
             theta = numpy.arange(180.)
             logSmPA = splev(theta,logspl)
             pylab.plot(theta,logSmPA)
             med = theta.copy()*0. +median
             pylab.plot(theta,numpy.log(med))
                
         pylab.xlim(-180.,180.)
         pylab.xlabel(r'$\theta$',fontsize=15)
         pylab.ylabel(r'$\langle \ell^%d C_\ell (\theta)\rangle$'%powerOfL,\
                      fontsize=15)
         pylab.show()
             
     
     
     if fitSpline:
         logspl = logspl
         threshold = median
         return binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
     else:
         return binnedPA, binnedTheta, binCount, binStdDev
Esempio n. 2
0
def init(mode):
    print("检测tunasync是否存在...")
    code = 1
    cmd = delegator.run("/usr/bin/tunasync -v")
    print(cmd.out)
    if cmd.return_code != 0:
        code = 0
    cmd = delegator.run("/usr/bin/tunasynctl -v")
    print(cmd.out)
    if cmd.return_code != 0:
        code = 0
    if code == 1:
        print("tunasync已存在,无需安装...")
    else:
        if mode:
            mode = int(mode)
        else:
            mode = 2
        print("开始安装...")
        try:
            if mode == 1:
                if bin() < 0:
                    print(
                        "安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
                    )
                    return 0
                else:
                    pass
            else:
                if build() < 0:
                    print(
                        "安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
                    )
                    return 0
                else:
                    pass
        except:
            print(
                "安装tunasync时出错\n您可以自行将tunasync和tunasynctl放置到/usr/bin目录后再进行初始化操作..."
            )
            return 0
    try:
        print("开始设置manager...")
        init_manager()
        print("开始设置服务...")
        systemd()
        print("启动manager...")
        systemd_control('start', 'manager')
        print("设置manager自启...")
        systemd_control('enable', 'manager')
        print("开始设置worker...")
        init_worker()
        # print("启动worker...")
        # systemd_control('start', 'worker')
        # print("设置worker自启...")
        # systemd_control('enable', 'worker')
        print("worker将不会启动直到新增第一个mirror...")
    except:
        print("设置时出错...")
Esempio n. 3
0
 def powerVsThetaInAnnulus(self,lLower,lUpper,deltaTheta=2.0,powerOfL=0,\
                           fitSpline=False,show=False,cutByMask=False):
     """
     @brief Given an anuulus, radially averages the power spectrum to produce a
     function of angle theta \f$P(\theta)\f$
     @param lLower Lower bound of the annulus
     @param lUpper Upper bound of the annulus
     @param deltaTheta Width of bins in theta
     @param powerOfL The power of L to multiply to PS with, before radial averaging
     @param fitspline If True returns a spline fit to the function
     @param show If True displays the function \f$P(\theta)\f$ 
     @param cutByMask If a kMask exists with p2d, then skip over masked pixels
     
     @return (If fitSpline:) binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
     
     @return (else:) binnedPA, binnedTheta, binCount, binStdDev 
     """
     if not(cutByMask):
         a= (self.modLMap < lUpper)
         b= (self.modLMap > lLower)
     else:
         a= ((self.modLMap < lUpper)*(self.kMask>0))
         b= ((self.modLMap > lLower)*(self.kMask>0))
     c = a*b
     
     indices = np.where(c)
     p = self.powerMap*self.modLMap**powerOfL
     thetaAnnu = np.ravel(self.thetaMap[indices])
     powerAnnu = (np.ravel(p[indices]))
     
     binnedTheta, binnedPA,binStdDev,binCount = utils.bin(thetaAnnu,powerAnnu,deltaTheta)
     
     if fitSpline:
         logspl = splrep(binnedTheta,np.log(np.abs(binnedPA)),s=2,k=4)
         median = np.median(np.abs(binnedPA))
         
     if show:
         pylab.plot(binnedTheta,np.log(np.abs(binnedPA)),'ro')
         if fitSpline:
             theta = np.arange(180.)
             logSmPA = splev(theta,logspl)
             pylab.plot(theta,logSmPA)
             med = theta.copy()*0. +median
             pylab.plot(theta,np.log(med))
                
         pylab.xlim(-180.,180.)
         pylab.xlabel(r'$\theta$',fontsize=15)
         pylab.ylabel(r'$\langle \ell^%d C_\ell (\theta)\rangle$'%powerOfL,\
                      fontsize=15)
         pylab.show()
             
     
     
     if fitSpline:
         logspl = logspl
         threshold = median
         return binnedPA, binnedTheta, binCount, binStdDev,logspl,threshold
     else:
         return binnedPA, binnedTheta, binCount, binStdDev
Esempio n. 4
0
def evaluate(function, calibrator, dist, n):
    zs = dist(size=n)
    ps = function(zs)
    phats = calibrator.calibrate(zs)
    bins = utils.get_discrete_bins(phats)
    data = list(zip(phats, ps))
    binned_data = utils.bin(data, bins)
    return utils.plugin_ce(binned_data)**2
def eval_top_calibration(probs, logits, labels, plugin=True):
    correct = (utils.get_top_predictions(logits) == labels)
    data = list(zip(probs, correct))
    bins = utils.get_discrete_bins(probs)
    binned_data = utils.bin(data, bins)
    if plugin:
        return utils.plugin_ce(binned_data)**2
    else:
        return utils.improved_unbiased_square_ce(binned_data)
Esempio n. 6
0
def polar_dot_binned(thetas,
                     at=1,
                     color='C0',
                     spread=0.25,
                     num_bins=50,
                     ax=None,
                     **kwargs):
    """Plots histogrammed/binned points onto the outside of a polar Axes.

    Parameters
    ----------
    thetas : 1D array-like
        Angular values to plot.

    at : int or float (optional, default=1)
        Position (r-value, along radial axis) to plot points.

    color : valid matplotlib color (optional, default='C0')
        Color of dots.

    spread : float (optional, default=2.)
        How spread out should the dots within the histogram bin be?

    num_bins : int (optional, default=50)
        How many bins should be used for generating the histogram?

    ax : matplotlib Axes handle (optional, default=None)
        Axes to plot dots. If `ax` is passed, make sure that it is already
        formatted in polar coordinates!

    **kwargs : keyword arguments to be passed to ax.scatter()

    Returns
    -------
    fig, ax : matplotlib Figure & Axes handle.
    """
    thetas = np.asarray(thetas)

    rr, tt = utils.bin(thetas,
                       at=at,
                       num_bins=num_bins,
                       align='left',
                       spread=spread)

    if ax is None:
        fig, ax = plt.subplots(subplot_kw=dict(polar=True))
    else:
        fig = plt.gcf()

    ax.scatter(tt, rr, color=color, **kwargs)

    return fig, ax
def eval_marginal_calibration(probs, logits, labels, plugin=True):
    ces = []  # Compute the calibration error per class, then take the average.
    k = logits.shape[1]
    labels_one_hot = utils.get_labels_one_hot(np.array(labels), k)
    for c in range(k):
        probs_c = probs[:, c]
        labels_c = labels_one_hot[:, c]
        data_c = list(zip(probs_c, labels_c))
        bins_c = utils.get_discrete_bins(probs_c)
        binned_data_c = utils.bin(data_c, bins_c)
        ce_c = utils.plugin_ce(binned_data_c)**2
        ces.append(ce_c)
    return np.mean(ces)
Esempio n. 8
0
 def get_mse_ce(logits, labels, ce_est):
     mses = []
     ces = []
     logits = np.array(logits)
     labels = np.array(labels)
     for l in range(k):
         cal_logits_l = platt_binners_equal_points[l](platts[l](
             logits[:, l]))
         data = list(zip(cal_logits_l, labels[:, l]))
         bins_l = utils.get_discrete_bins(cal_logits_l)
         binned_data = utils.bin(data, bins_l)
         # probs = platts[l](logits[:, l])
         # for p in [1, 5, 10, 20, 50, 85, 88.5, 92, 94, 96, 98, 100]:
         #     print(np.percentile(probs, p))
         # import time
         # time.sleep(100)
         # print('lengths')
         # print([len(d) for d in binned_data])
         ces.append(ce_est(binned_data))
         mses.append(
             np.mean([(prob - label)**2 for prob, label in data]))
     return np.mean(mses), np.mean(ces)
 def estimator(data):
     binned_data = utils.bin(data, bins)
     return utils.plugin_ce(binned_data, power=lp)
Esempio n. 10
0
n_scrapes = 10
for i in range(n_scrapes):
    tweets = twitter.search(q='#monday')
    time.sleep(3)
    print "we are on iteration %i of %i"%(i+1, n_scrapes)
    for key in tweets['statuses']:
        mywords.append(key['text'].encode("ascii","ignore"))
        for word in (key['text']).split():
            wordLength.append(len(word))

# mywords = mywords.encode("ascii", "ignore")
tags = make_tags(get_tag_counts(mywords), maxsize=80)

create_tag_image(tags, 'cloud_large.png', size=(900, 600), fontname='Lobster')

d = makeWordDict(mywords)
# print d
# for key in d:
#     print key , d[key]


# wordLength = sorted(wordLength)
a,b, = bin(inputData = wordLength, binWidth = 1)
# # print wordLength
# 
# 
makePlot(b,a)
# 
# a_1, b_1 = bin(inputData=wordLength, binWidth = 5)
# makePlot(b_1,a_1)
 def estimator(data):
     binned_data = utils.bin(data, bins)
     return estimators[i](binned_data)
def compare_estimators(logits,
                       labels,
                       platt_data_size,
                       bin_data_size,
                       num_bins,
                       ver_base_size=2000,
                       ver_size_increment=1000,
                       num_resamples=100,
                       save_name='cmp_est'):
    # Convert logits to prediction, probs.
    predictions = utils.get_top_predictions(logits)
    probs = utils.get_top_probs(logits)
    correct = (predictions == labels)
    # Platt scale on first chunk of data
    platt = utils.get_platt_scaler(probs[:platt_data_size],
                                   correct[:platt_data_size])
    platt_probs = platt(probs)
    estimator_names = ['biased', 'unbiased']
    estimators = [
        lambda x: utils.plugin_ce(x)**2, utils.improved_unbiased_square_ce
    ]

    bins = utils.get_equal_bins(platt_probs[:platt_data_size + bin_data_size],
                                num_bins=num_bins)
    binner = utils.get_discrete_calibrator(
        platt_probs[platt_data_size:platt_data_size + bin_data_size], bins)
    verification_probs = binner(platt_probs[platt_data_size + bin_data_size:])
    verification_correct = correct[platt_data_size + bin_data_size:]
    verification_data = list(zip(verification_probs, verification_correct))
    verification_sizes = list(
        range(ver_base_size,
              len(verification_probs) + 1, ver_size_increment))
    # We want to compare the two estimators when varying the number of samples.
    # However, a single point of comparison does not tell us much about the estimators.
    # So we use resampling - we resample from the test set many times, and run the estimators
    # on the resamples. We stores these values. This gives us a sense of the range of values
    # the estimator might output.
    # So estimates[i][j][k] stores the estimate when using estimator i, with verification_sizes[j]
    # samples, in the k-th resampling.
    estimates = np.zeros(
        (len(estimators), len(verification_sizes), num_resamples))
    # We also store the certified estimates. These represent the upper bounds we get using
    # each estimator. They are computing using the std-dev of the estimator estimated by
    # Bootstrap.
    cert_estimates = np.zeros(
        (len(estimators), len(verification_sizes), num_resamples))
    for ver_idx, verification_size in zip(range(len(verification_sizes)),
                                          verification_sizes):
        for k in range(num_resamples):
            # Resample
            indices = np.random.choice(list(range(len(verification_data))),
                                       size=verification_size,
                                       replace=True)
            cur_verification_data = [verification_data[i] for i in indices]
            cur_verification_probs = [verification_probs[i] for i in indices]
            bins = utils.get_discrete_bins(cur_verification_probs)
            # Compute estimates for each estimator.
            for i in range(len(estimators)):

                def estimator(data):
                    binned_data = utils.bin(data, bins)
                    return estimators[i](binned_data)

                cur_estimate = estimator(cur_verification_data)
                estimates[i][ver_idx][k] = cur_estimate
                # cert_resampling_estimates[j].append(
                # 	cur_estimate + utils.bootstrap_std(cur_verification_data, estimator, num_samples=20))

    estimates = np.sort(estimates, axis=-1)
    lower_bound = int(0.1 * num_resamples)
    median = int(0.5 * num_resamples)
    upper_bound = int(0.9 * num_resamples)
    lower_estimates = estimates[:, :, lower_bound]
    upper_estimates = estimates[:, :, upper_bound]
    median_estimates = estimates[:, :, median]

    # We can also compute the MSEs of the estimator.
    bins = utils.get_discrete_bins(verification_probs)
    true_calibration = utils.plugin_ce(utils.bin(verification_data, bins))**2
    print(true_calibration)
    print(np.sqrt(np.mean(estimates[1, -1, :])))
    errors = np.abs(estimates - true_calibration)
    accumulated_errors = np.mean(errors, axis=-1)
    error_bars_90 = 1.645 * np.std(errors, axis=-1) / np.sqrt(num_resamples)
    print(accumulated_errors)
    plt.errorbar(verification_sizes,
                 accumulated_errors[0],
                 yerr=[error_bars_90[0], error_bars_90[0]],
                 barsabove=True,
                 color='red',
                 capsize=4,
                 label='plugin')
    plt.errorbar(verification_sizes,
                 accumulated_errors[1],
                 yerr=[error_bars_90[1], error_bars_90[1]],
                 barsabove=True,
                 color='blue',
                 capsize=4,
                 label='debiased')
    plt.ylabel("Mean-Squared-Error")
    plt.xlabel("Number of Samples")
    plt.legend(loc='upper right')
    plt.show()

    plt.ylabel("Number of estimates")
    plt.xlabel("Absolute deviation from ground truth")
    bins = np.linspace(np.min(errors[:, 0, :]), np.max(errors[:, 0, :]), 40)
    plt.hist(errors[0][0], bins, alpha=0.5, label='plugin')
    plt.hist(errors[1][0], bins, alpha=0.5, label='debiased')
    plt.legend(loc='upper right')
    plt.gca().yaxis.set_major_formatter(PercentFormatter(xmax=num_resamples))
    plt.show()
Esempio n. 13
0
def f():
  s = 0
  for x in xrange(1,1000000):
    if isPal(str(x)) and isPal(bin(x)): s+=x
  return s
Esempio n. 14
0
def eval_top_calibration(probs, logits, labels):
    correct = (utils.get_top_predictions(logits) == labels)
    data = list(zip(probs, correct))
    bins = utils.get_discrete_bins(probs)
    binned_data = utils.bin(data, bins)
    return utils.plugin_ce(binned_data)**2
def striphist(
    data,
    order=None,
    markerfacecolors=None,
    markeredgecolors=None,
    at=None,
    num_bins=50,
    align='center',
    spread=0.25,
    ax=None,
    **kwargs
    ):
    """Strip chart with histogrammed point visualization.

    Parameters
    ----------
    data : dictionary
        Data to plot. Keys should be string, vals should be 1D array-like.
    
    order : list of string or None
        Each string should be a valid key in `data`. This is the order to plot
        each group along the x-axis. If None, groups will be plotted
        alphabetically from left to right.

    markerfacecolors, markeredgecolors : list of valid matplotlib colors or None
        Marker face and edge colors of each group to plot. Should be ordered as 
        in `order`. If None, current matplotlib color cycle will be used to 
        select colors.
    
    at : list of int or float or None
        Where to plot each group along the x-axis. Should be ordered as in 
        `order`.

    num_bins : int
        How many bins to use for histogramming/binning data.

    align : string
        How to display histogrammed points. Options are 'center' or 'left'.

    spread : float
        How spread out should the histogrammed points be?

    ax : matplotlib Axes handle or None
        Axes handle for plotting. If None, one will be generated for you.

    Parameters
    ----------
    **kwargs : keyword arguments to be passed to ax.scatter().

    Returns
    -------
    fig, ax : matplotlib Figure & Axes handle.
    """
    if order is None:
        order = sorted(data.keys())

    if markerfacecolors is None:
        markerfacecolors = ['C{}'.format(i % 10) for i in xrange(len(data))]
    if markeredgecolors is None:
        markeredgecolors = ['C{}'.format(i % 10) for i in xrange(len(data))]

    if at is None:
        at = range(1, len(data) + 1)

    if ax is None:
        fig, ax = plt.subplots()
    else:
        fig = plt.gcf()

    for i, group_name in enumerate(order):
        xx, yy = utils.bin(data[group_name], at=at[i], num_bins=num_bins,
            align=align, spread=spread)

        kwargs.update(
            {
                'c': markerfacecolors[i],
                'edgecolors': markeredgecolors[i]
            }
        )

        ax.scatter(xx, yy, **kwargs)

    return fig, ax