def test_nanstd_roundoff(self): # Regression test for GH 10242 (test data taken from GH 10489). Ensure # that variance is stable. data = Series(766897346 * np.ones(10)) for ddof in range(3): result = data.std(ddof=ddof) assert result == 0.0
def aggregate_features(text): f = collections.OrderedDict() f['questions'] = text.count('?') f['exclamation'] = text.count('!') f['commas'] = text.count(',') tb = TextBlob(text) sentences = tb.sentences f['number of sentences'] = len(sentences) f['number of words'] = len(tb.words) lengths = Series([len(s) for s in sentences]) f['length_std'] = lengths.std() if f['number of sentences'] > 1 else 0 share_stopwords = Series([(w in cachedStopWords) for w in tb.words]).mean() f['share stopwords'] = share_stopwords certain_words = ['sure', 'certain', 'undoubt', 'clear', 'absolute'] certain_number = Series([len(re.findall(cw, text)) for cw in certain_words]).sum() f['certain_share'] = certain_number / f['number of words'] sentiment = TextBlob(text).sentiment._asdict() f.update(sentiment) return f
def test_nanstd_roundoff(self): # Regression test for GH 10242 (test data taken from GH 10489). Ensure # that variance is stable. data = Series(766897346 * np.ones(10)) for ddof in range(3): result = data.std(ddof=ddof) self.assertEqual(result, 0.0)
def _check_Xy(X: pd.DataFrame, y: pd.Series, *, norm_y=False) -> Tuple[pd.Series, pd.Series]: if np.ndim(X) == 1: X = pd.Series(X).to_frame() elif np.ndim(X) == 2: X = pd.DataFrame(X) assert X.ndim == 2 assert np.ndim(y) == 1 assert len(X) == len(y) valid = ~X.isnull().any(1).values X = pd.Series(list(zip(*X.values[valid].T)), name=tuple(X.columns)).astype('category') y = pd.Series(y).reset_index(drop=True)[valid] if is_object_dtype(y): y = pd.Categorical(y) if norm_y: assert is_numeric_dtype(y) y = (y - y.mean()) / y.std() return X, y
def plot_norm(data: pd.Series, bins=10, ax=None, is_show_plot=None): """ 显示当前数据的正太分布曲线 :param data: :param bins: bar 数量 :param ax: 如果为None,则新建一个画布 :param is_show_plot: 是否展示 :return: n, bins_v, mean, std """ if ax is None: fig, ax = plt.subplots() if is_show_plot is None: is_show_plot = True if is_show_plot is None: is_show_plot = False n, bins_v = np.histogram(data, bins=bins) mu = data.mean() # mean of distribution sigma = data.std() # standard deviation of distribution # def norm_func(x, mu, sigma): # pdf = np.exp(-((x - mu)**2)/(2*sigma**2)) / (sigma * np.sqrt(2*np.pi)) # return pdf # y = norm_func(bins, mu, sigma) # 与 mlab.normpdf(bins, mu, sigma) 相同 # y = mlab.normpdf(bins, mu, sigma) y = stats.norm.pdf(bins_v, loc=mu, scale=sigma) ax.plot(bins_v, y, '--') plt.grid(True) if is_show_plot: plt.show() return n, bins_v, mu, sigma
def _predict_core(self, s: pd.Series) -> pd.Series: mean = s.mean() std = s.std() if std == 0: std = 1 return (s - mean) / std
def __init__(self, col: Series): col: ndarray = col.to_numpy() self._min: number = col.min(initial=None) self._max: number = col.max(initial=None) self._range: number = self._max - self._min self._mean: number = col.mean() self._std: number = col.std()
def kurtosis(returns: pd.Series): """ Computes the skewness of a asset series """ relative_ret = returns - returns.mean() sigma = returns.std(ddof=0) exp = (relative_ret**4).mean() return exp / sigma**4
def fill_outliers(col: pd.Series): """ Remove outliers of each col """ mean, std = col.mean(), col.std() upper, lower = mean + 3 * std, mean - 3 * std col[col > upper] = np.floor(upper) col[col < lower] = np.floor(lower) return col.values
def calculate_annualized_volatility(return_series: pd.Series) -> float: """ Calculates annualized volatility for a date-indexed return series. Works for any interval of date-indexed prices and returns. """ years_past = get_years_past(return_series) entries_per_year = return_series.shape[0] / years_past return return_series.std() * np.sqrt(entries_per_year)
def _mask_outliers(vec: pd.Series, stdv_times): vec_mean = vec.mean() vec_stdv = vec.std() upper = vec_mean + vec_stdv * stdv_times lower = vec_mean - vec_stdv * stdv_times vec[((lower > vec) | (vec > upper))] = np.nan return vec
def before_after_3sigma(data: pd.Series) -> pd.Series: miu = data.mean() sigma = data.std() threshold_down = miu - 3 * sigma threshold_up = miu + 3 * sigma data[data.ge(threshold_up)] = threshold_up data[data.le(threshold_down)] = threshold_down return data
def from_series(feature_name: str, series: Series): """从pandas.Series中构造""" assert types.is_numeric_dtype(series), series.dtypes return NumericColumn(feature_name=feature_name, min_value=series.min(), max_value=series.max(), mean_value=series.mean(), std_value=series.std())
def sharpe_ratio(corrs: pd.Series) -> np.float32: """ Calculate the Sharpe ratio for Numerai by using grouped per-era data :param corrs: A Pandas Series containing the Spearman correlations for each era :return: A float denoting the Sharpe ratio of your predictions. """ return corrs.mean() / corrs.std()
def fit_transform_normalize_y(y_train): y_train_bx, _ = boxcox(y_train) y_train_bx = Series(y_train_bx, y_train.index) outliers = tukey_outlier_test(y_train_bx) y_train_outliers_removed = y_train.drop(outliers) y_train_bx, bx_lambda = boxcox(y_train_outliers_removed) y_train_bx = Series(y_train_bx, y_train_outliers_removed.index) y_train_bx_whitened = (y_train_bx - y_train_bx.mean()) / y_train_bx.std() return y_train_bx_whitened, bx_lambda
def roys_safety_first_criterion(portfolio_returns: pd.Series, minimum_threshold=0.02, period=252): """ :param portfolio_returns: Pandas series or dataframe representing percentage changes of the security (or portfolio) returns over time. It should be same time range and frequency as risk free rates :param minimum_threshold: minimum acceptable return, below which the returns are less desirable. :param period: period to compute statistics of returns for. For instance, to compute yearly, then input 252, and to compute monthly, then input 21. :return: """ return (portfolio_returns.mean() * period - minimum_threshold) / (portfolio_returns.std() * math.sqrt(period))
def fit(self, x: pd.Series): if self.method == "Gaussian": self.mean, self.std = x.mean(), x.std() elif self.method == "RankGaussian": # TODO: store state pass elif self.method == "MinMax": self.min, self.max = x.min(), x.max() return self
def stdev(items): """Calculate stdev from the items. :param items: Stdev is calculated from these items. :type items: list :returns: Stdev. :rtype: float """ return Series.std(Series(items))
def filter_outliers(data: pd.Series, std: int=3) -> pd.Series: """ Remove outliers from ``Series``. :param data: data to be filtered :param std: number of standard deviations to be filtered. Default is 3 :return: ``Series`` filtered out of outliers """ return data[(data - data.mean()).abs() <= (std * data.std())]
def z_score(data: pd.Series): """ :param data: :return: """ miu = data.mean() sigma = data.std() stand = (data - miu) / sigma return stand
def test_describe_ints(self): ser = Series([0, 1, 2, 3, 4], name="int_data") result = ser.describe() expected = Series( [5, 2, ser.std(), 0, 1, 2, 3, 4], name="int_data", index=["count", "mean", "std", "min", "25%", "50%", "75%", "max"], ) tm.assert_series_equal(result, expected)
def winsorize(x: pd.Series, limit: float = 2.5, w: int = 0) -> pd.Series: """ Limit extreme values in series :param x: time series of prices :param limit: max z-score of values :param w: window: number of observations to use (defaults to length of series) :return: timeseries of winsorized values **Usage** Cap and floor values in the series which have a z-score greater or less than provided value. This function will restrict the distribution of values. Calculates the sample standard deviation and adjusts values which fall outside the specified range to be equal to the upper or lower limits Lower and upper limits are defined as: :math:`upper = \mu + \sigma \\times limit` :math:`lower = \mu - \sigma \\times limit` Where :math:`\mu` and :math:`\sigma` are sample mean and standard deviation. The series is restricted by: :math:`R_t = max( min( X_t, upper), lower )` See `winsorising <https://en.wikipedia.org/wiki/Winsorizing>`_ for additional information **Examples** Generate price series and winsorize z-score of returns over :math:`22` observations >>> prices = generate_series(100) >>> winsorize(zscore(returns(prices), 22)) **See also** :func:`zscore` :func:`mean` :func:`std` """ w = w or x.size if x.size < 1: return x assert w, "window is not 0" mu = x.mean() sigma = x.std() high = mu + sigma * limit low = mu - sigma * limit ret = ceil(x, high) ret = floor(ret, low) return ret
def CZI(x: pd.Series): """ Pearson Type III distribution """ zsi = (x - x.mean()) / x.std() cs = np.power(zsi, 3) / len(x) czi = ( 6.0 / cs * np.power( (cs / 2.0 * zsi + 1.0), 1.0 / 3.0 ) - 6.0 / cs + cs / 6.0 ) return czi
def calc_value_at_risk(returns: pd.Series, alpha=0.95, hist=True) -> float: """ Value at risk in % """ if hist: return returns.quantile(q=1 - alpha) else: mu = returns.mean() sigma = returns.std() return norm.ppf(1 - alpha, mu, sigma)
def _bigger_than_cutoff(scores: pd.Series) -> pd.Series: """Identifies all the values that have a similarity scores bigger than the first standard deviation from the mean. If they are all identical we return everything. """ if scores.mean() == 1.0: return scores dev = scores.std() most_similar = scores[scores > dev] return most_similar
def std(grouped_data: pd.Series) -> float: """ Extract standard deviation of a given pandas Series :param grouped_data: grouped data :type grouped_data: pd.Series :return: standard deviation value :rtype: float """ return grouped_data.std()
def annualized_vol(r: pd.Series, periods_per_year): """ Inputs: r:pd.Series or pd.DataFrame periods_per_year = frequency of return in a year If data has daily return value then periods_per_year = 365 or 252 If data has monthly return value then periods_per_year = 12 Output: Returns annualized Volatility """ return r.std() * (periods_per_year**0.5)
def removeOutliers(S: pd.Series, inplace=False, printOutput=False): result = S = S[~((S - S.mean()).abs() > 3 * S.std())] if printOutput: print('S Length: ', getRowsNum(S)) print('results Length: ', getRowsNum(result)) if inplace: S = result return None else: return result
def normalizer(series: pd.Series): std = series.std() mean = series.mean() def rescale(v): return v * std + mean def scale(v): return (v - mean) / std return (scale, rescale)
def fuzzy_smf(series: pd.Series): avg_avf = series.mean() sd_avf = series.std() max_avf = series.max() # calculate a a = calc_a(avg_avf, sd_avf, max_avf) # calculate c c = calc_c(avg_avf, sd_avf, max_avf) return smf(series, a, c)
def whole_number_digits(series: pd.Series) -> int: """Determine the relevant number of whole number digits for the given series. The standard deviation, as a measure of the variation of the data, is used. Args: series: A Pandas Series (of floats) Returns: The number of whole number digits associated with this series """ stddev = series.std(ddof=0) digits = int(math.log10(stddev)) + 1 return digits
def z_transform(x: pd.Series) -> pd.Series: """ Z-transforms scores. Args: x: values to Z-transform Returns: Z-transformed scores """ return (x - x.mean()) / x.std()
def sharpeRatio(self, returns: pd.Series): """ Sets the Sharpe Ratio for the Asset :param returns: Series that contains the returns of the Asset """ if not isinstance(returns, pd.Series): raise TypeError(classmethod.__name__ + ': returns must be a Pandas Series.' + '. Current type of the returns: ' + str(type(returns))) self._sharpeRatio = (returns.mean() / returns.std()) * (len(returns)** 0.5)
def get_summary_indicators_from_hist(sf, hist, int_index=False): seriesHist = Series(hist) maxs = { 'freq': dict() } means = {'freq': seriesHist.mean()} medians = {'freq': seriesHist.median()} stds = {'freq': seriesHist.std()} maxs['freq']['freq'] = seriesHist.max() maxs['freq']['index'] = seriesHist.idxmax() index_total = 'NA' if int_index: index = seriesHist.index index = index.astype(int) index_list = index.tolist() index_total = sum([seriesHist[i] * index_list[i] for i in range(len(index_list))]) index_series = Series(index_list) means['index'] = index_series.mean() medians['index'] = index_series.median() stds['index'] = index_series.std() maxs['freq']['index'] = int(maxs['freq']['index']) maxs['index'] = dict() maxs['index']['index'] = max(index_list) maxs['index']['freq'] = hist[str(maxs['index']['index'])] return { 'means': means, 'medians': medians, 'stds': stds, 'max': maxs, 'index_total': index_total }
k_list = list() size = len(new_list) k_sum = 0 for i in range(size): if i == 0: #k_sum = float(new_list[0][1]) continue k_sum = k_sum + (float(new_list[i-1][1])/float(new_list[i][0])) k_list.append(float(new_list[i-1][1])/float(new_list[i][0])) k_avg = k_sum/size #consu_ser = Series(data=k_list,index=range(len(k_list))) consu_ser = Series(data=k_list,index=range(len(k_list))) #print consu_ser.describe() mean = consu_ser.mean() std_dev = consu_ser.std() modified_list = list() for i in range(len(k_list)): if (k_list[i] < mean + std_dev) and (k_list[i] > mean - std_dev): modified_list.append(k_list[i]) plt.hist(modified_list) plt.show() consu_ser_mod = Series(data=modified_list,index=range(len(modified_list))) k_avg = consu_ser_mod.sum()/len(modified_list) thresh_sum = 0 for i in range(size): if i == 0: continue thresh_sum = thresh_sum + float(new_list[i-1][1]) - float(new_list[i][0])*k_avg
account.password ) traderChannel #%% 定长队列 from collections import deque q = deque(maxlen=10) for i in range(10): q.append(i) #%% from pandas import Series s = Series(range(10)) s.mean() s.std() #%% 交易结果导出到excel import os os.chdir('/home/duhan/github/CTPTrader') from comhelper import setDjangoEnvironment setDjangoEnvironment() from database.models import * from django_pandas.io import read_frame from pandas.io.excel import ExcelWriter df = read_frame(ModelPosition.objects.filter(state='close')) writer = ExcelWriter('/tmp/output.xls') df.to_excel(writer) writer.save()
def adfTest(spread): ADF_p_value = ts.adfuller(spread, 1)[1] return ADF_p_value ADF_p_value = adfTest(spread) if ADF_p_value <= 0.05: print "The spread is likely mean-reverting." else: print "The spread is not mean-reverting." spread = Series(spread) signalMean = spread.mean() signalDev = spread.std() openMult = 1.0 closeMult = 0.5 stopLossMult = 4.0 openSignal = signalDev * openMult; closeSignal = signalDev * closeMult; stopLossSignal = signalDev * stopLossMult; residSpread = spread - signalMean residSpread.plot() openSignalUp = openSignal * (residSpread * 0 + 1) openSignalDown = -openSignal * (residSpread * 0 + 1) openSignalUp.plot()
data2.tail() # summary statistics # d1990.sum() data.describe() data.std() # how much did total population change between 1990 and 2010? for i in [1990,2010]: total = sum(data['pop'][data['yr']==i]) # +42,099,904 # TODO: how many people did the average congressperson represent in 1990? # we could also represent a single variable as a series with hierarchical indexing p = Series(data['pop'].values, index=[data['st'], data['yr']]) p['North Carolina'] p.mean(level='st') p.std() # TODO: calculate standard deviation by year p.swaplevel('st', 'yr') # correlation data['pop'].corr(data['ev']) # estimate a linear model model = pd.ols(y=data['ev'], x=data['popm']) print model model.beta