def run_once(self): wallets = get_wallets() my_usd = int(wallets['USD']['Balance']['value_int']) my_btc = int(wallets['BTC']['Balance']['value_int']) curr_ask = current_ask_price() curr_bid = current_bid_price() self.prices.append(pd.DataFrame([curr_ask, curr_bid], columns=['ask', 'bid'])) if pd.rolling_count(self.prices['ask']) < self.lookback: print 'only ', pd.rolling_count(self.prices['ask']), ' observations so far' return means = pd.rolling_mean(self.prices, self.lookback, min_periods = self.lookback) stddev = pd.rolling_std(self.prices, self.lookback, min_periods = self.lookback) lower = means - stddev upper = means + stddev normalized = (close - means) / stddev timestamps = self.prices.index # should we sell? bollinger_ask_now = normalized['ask'].ix[ldt_timestamps[len(timestamps) - 1]] bollinger_ask_last = normalized['ask'].ix[ldt_timestamps[len(timestamps) - 2]] if bollinger_ask_now <= -2.0 and bollinger_ask_last >= -2.0: print now(), 'begin sell ', my_btc print 'sell result', sell(my_btc) # should we buy? bollinger_bid_now = normalized['bid'].ix[ldt_timestamps[len(timestamps) - 1]] bollinger_bid_last = normalized['bid'].ix[ldt_timestamps[len(timestamps) - 2]] if bollinger_bid_now >= 2.0 and bollinger_bid_last <= 2.0: amount = int(my_usd / curr_bid) print now(), 'begin buy ', amount print 'sell result', buy(amount)
def a_counter(grouped):
se = grouped.set_index('ACCIDENT_DT')['DEGREE_INJURY_CD']
# se is the time series of accident dates restricted to a single MINE_ID
se = se.resample("D")
df = pd.DataFrame({'degree_injury_cd':se, 'a90':pd.rolling_count(se, 90), 'a30':pd.rolling_count(se, 30)})
# TODO: include a sum of injury counts by day
return df
def test_filter_1(self):
df = self.fetcher.fetch_window('close', DATES[self.si-window: self.ei+1])
df = pd.rolling_sum(df.fillna(0), window) > 2 * pd.rolling_count(df, window)
df1 = df.shift(1).iloc[window:].astype(bool)
df2 = self.filter.filter(self.startdate, self.enddate)
print 'bm', df1.sum(axis=1)
self.assertTrue(frames_equal(df1, df2))
def rolling_functions_tests(p, d):
# Old-fashioned rolling API
assert_eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
assert_eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
assert_eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
assert_eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
assert_eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
assert_eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
assert_eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
assert_eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
# see note around test_rolling_dataframe for logic concerning precision
assert_eq(pd.rolling_skew(p, 3),
dd.rolling_skew(d, 3), check_less_precise=True)
assert_eq(pd.rolling_kurt(p, 3),
dd.rolling_kurt(d, 3), check_less_precise=True)
assert_eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
assert_eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
with ignoring(ImportError):
assert_eq(pd.rolling_window(p, 3, 'boxcar'),
dd.rolling_window(d, 3, 'boxcar'))
# Test with edge-case window sizes
assert_eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
assert_eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
assert_eq(pd.rolling_sum(p, 3, min_periods=3),
dd.rolling_sum(d, 3, min_periods=3))
def collection_freq(breath_df, win):
print(breath_df.columns)
for ds_type in ['ds', 'pl', 'pvt', 'ie']:
breath_df['{0}_rolling'.format(ds_type)] = pd.rolling_sum(breath_df['analysis.' + ds_type], window = 60 * win,
center = True, min_periods = 1)
breath_df[ds_type + '_tot_rolling'] = pd.rolling_count(breath_df['analysis.' + ds_type], window = 60 * win,
center = True)
breath_df[ds_type + '_freq'] = breath_df[ds_type + '_rolling'] / breath_df[ds_type + '_tot_rolling']
# add rolling average for Fio2, PEEP, p_mean
try:
breath_df['peep_rolling'] = pd.rolling_mean(breath_df['vent_settings.PEEP'], window = 60 * win,
center = True, min_periods = 1)
except KeyError:
pass
try:
breath_df['p_mean_rolling'] = pd.rolling_mean(breath_df['vent_settings.p_mean'], window = 60 * win,
center = True, min_periods = 1)
except KeyError:
pass
try:
breath_df['fio2_rolling'] = pd.rolling_mean(breath_df['vent_settings.FiO2'], window = 60 * win,
center = True, min_periods = 1)
except KeyError:
pass
return breath_df
def rollingStats(self, selectCol = [], splitCol=None, sepCol=None, startTime=None, endTime=None, window=60, quantile=0.1, freq='10s', min_periods=5 ):
df = self.dfSetup()
## Selects a list of columns to use and splits a column into single type if it contains more than one
# eg. if a file contains multiple sensor readings
if (len(selectCol) > 0):
dfSub = df[selectCol]
else:
dfSub = df
if (splitCol and sepCol):
dfSub = dfSub[dfSub[splitCol] == sepCol]
## Converts datetime column to datatime object index, then use it to create time slices
# Time format '2015-10-17 09:00:00' May use the dfOther to use other data frames
if (startTime and endTime):
dfSub = dfSub[ startTime : endTime ]
else:
dfSub = dfSub
if (splitCol):
dfSub = dfSub.drop(splitCol, axis=1) # Remove columns used to split entries
valueName = dfSub.columns.values[0]
outList = []
counts = pd.rolling_count(dfSub,window,freq=freq).rename(columns = {valueName:'rolling_counts'})
outList.append(counts)
means = pd.rolling_mean(dfSub, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_mean'})
outList.append(means)
rms = np.sqrt(pd.rolling_mean(dfSub**2, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_rms'}) )
outList.append(rms)
medians = pd.rolling_median(dfSub, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_median'})
outList.append(medians)
stds = pd.rolling_std(dfSub, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_std'})
outList.append(stds)
mins = pd.rolling_min(dfSub, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_min'})
outList.append(mins)
maxs = pd.rolling_max(dfSub, window, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_max'})
outList.append(maxs)
quants = pd.rolling_quantile(dfSub, window, quantile, min_periods=min_periods, freq=freq).rename(columns = {valueName:'rolling_quantile'})
outList.append(quants)
dfOut = pd.concat(outList, axis=1)
return dfOut
def test_filter_2(self):
df = self.fetcher.fetch_window('close', DATES[self.si-window: self.ei+1])
parent = df.notnull()
df = df.shift(1)
df[~parent] = None
df = pd.rolling_sum(df.fillna(0), window) > 2 * pd.rolling_count(df, window)
df[~parent] = False
df = df.iloc[window:]
self.assertTrue(frames_equal(df, self.filter.filter(self.startdate, self.enddate, parent)))
def calc_stats(timestamps,
gain,
pol='no polarizarion',
windowtime=1200,
minsamples=1):
""" calculate the Stats needed to evaluate the obsevation"""
returntext = []
gain_ts = pandas.Series(gain,
pandas.to_datetime(np.round(timestamps),
unit='s')).asfreq(freq='1s')
mean = pandas.rolling_mean(gain_ts, windowtime, minsamples)
std = pandas.rolling_std(gain_ts, windowtime, minsamples)
windowgainchange = std / mean * 100
dtrend_std = pandas.rolling_apply(gain_ts, windowtime, detrend, minsamples)
#trend_std = pandas.rolling_apply(ts,5,lambda x : np.ma.std(x-(np.arange(x.shape[0])*np.ma.polyfit(np.arange(x.shape[0]),x,1)[0])),1)
detrended_windowgainchange = dtrend_std / mean * 100
timeval = timestamps.max() - timestamps.min()
window_occ = pandas.rolling_count(gain_ts, windowtime) / float(windowtime)
#rms = np.sqrt((gain**2).mean())
returntext.append(
"Total time of obsevation : %f (seconds) with %i accumulations." %
(timeval, timestamps.shape[0]))
#returntext.append("The mean gain of %s is: %.5f"%(pol,gain.mean()))
#returntext.append("The Std. dev of the gain of %s is: %.5f"%(pol,gain.std()))
#returntext.append("The RMS of the gain of %s is : %.5f"%(pol,rms))
#returntext.append("The Percentage variation of %s is: %.5f"%(pol,gain.std()/gain.mean()*100))
returntext.append(
"The mean Percentage variation over %i seconds of %s is: %.5f (req < 2 )"
% (windowtime, pol, windowgainchange.mean()))
returntext.append(
"The Max Percentage variation over %i seconds of %s is: %.5f (req < 2 )"
% (windowtime, pol, windowgainchange.max()))
returntext.append(
"The mean detrended Percentage variation over %i seconds of %s is: %.5f (req < 2 )"
% (windowtime, pol, detrended_windowgainchange.mean()))
returntext.append(
"The Max detrended Percentage variation over %i seconds of %s is: %.5f (req < 2 )"
% (windowtime, pol, detrended_windowgainchange.max()))
#a - np.round(np.polyfit(b,a.T,1)[0,:,np.newaxis]*b + np.polyfit(b,a.T,1)[1,:,np.newaxis])
pltobj = plt.figure()
plt.title('Percentage Variation of %s pol, %i Second sliding Window' % (
pol,
windowtime,
))
windowgainchange.plot(label='Orignal')
detrended_windowgainchange.plot(label='Detrended')
window_occ.plot(label='Window Occupancy')
plt.hlines(2, timestamps.min(), timestamps.max(), colors='k')
plt.ylabel('Percentage Variation')
plt.xlabel('Date/Time')
plt.legend(loc='best')
#plt.title(" %s pol Gain"%(pol))
#plt.plot(windowgainchange.mean(),'b',label='20 Min (std/mean)')
#plt.plot(np.ones_like(windowgainchange.mean())*2.0,'r',label=' 2 level')
return returntext, pltobj # a plot would be cool
def test_filter_1(self):
df = self.fetcher.fetch_window('close',
DATES[self.si - window:self.ei + 1])
df = pd.rolling_sum(df.fillna(0),
window) > 2 * pd.rolling_count(df, window)
df1 = df.shift(1).iloc[window:].astype(bool)
df2 = self.filter.filter(self.startdate, self.enddate)
print 'bm', df1.sum(axis=1)
self.assertTrue(frames_equal(df1, df2))
def test_filter_2(self):
df = self.fetcher.fetch_window('close',
DATES[self.si - window:self.ei + 1])
parent = df.notnull()
df = df.shift(1)
df[~parent] = None
df = pd.rolling_sum(df.fillna(0),
window) > 2 * pd.rolling_count(df, window)
df[~parent] = False
df = df.iloc[window:]
self.assertTrue(
frames_equal(
df, self.filter.filter(self.startdate, self.enddate, parent)))
def rolling_tests(p, d):
eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3))
eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3))
eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
mad = lambda x: np.fabs(x - x.mean()).mean()
eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
eq(pd.rolling_window(p, 3, 'boxcar'), dd.rolling_window(d, 3, 'boxcar'))
# Test with edge-case window sizes
eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3, min_periods=3))
def rolling_functions_tests(p, d):
# Old-fashioned rolling API
eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3))
eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3))
eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
with ignoring(ImportError):
eq(pd.rolling_window(p, 3, 'boxcar'), dd.rolling_window(d, 3, 'boxcar'))
# Test with edge-case window sizes
eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3, min_periods=3))
def rolling_functions_tests(p, d):
# Old-fashioned rolling API
eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3))
eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3))
eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
with ignoring(ImportError):
eq(pd.rolling_window(p, 3, "boxcar"), dd.rolling_window(d, 3, "boxcar"))
# Test with edge-case window sizes
eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3, min_periods=3))
def rolling_tests(p, d):
eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3))
eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3))
eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
mad = lambda x: np.fabs(x - x.mean()).mean()
eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
with ignoring(ImportError):
eq(pd.rolling_window(p, 3, 'boxcar'),
dd.rolling_window(d, 3, 'boxcar'))
# Test with edge-case window sizes
eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3,
min_periods=3))
def rolling_functions_tests(p, d):
# Old-fashioned rolling API
eq(pd.rolling_count(p, 3), dd.rolling_count(d, 3))
eq(pd.rolling_sum(p, 3), dd.rolling_sum(d, 3))
eq(pd.rolling_mean(p, 3), dd.rolling_mean(d, 3))
eq(pd.rolling_median(p, 3), dd.rolling_median(d, 3))
eq(pd.rolling_min(p, 3), dd.rolling_min(d, 3))
eq(pd.rolling_max(p, 3), dd.rolling_max(d, 3))
eq(pd.rolling_std(p, 3), dd.rolling_std(d, 3))
eq(pd.rolling_var(p, 3), dd.rolling_var(d, 3))
# see note around test_rolling_dataframe for logic concerning precision
eq(pd.rolling_skew(p, 3), dd.rolling_skew(d, 3), check_less_precise=True)
eq(pd.rolling_kurt(p, 3), dd.rolling_kurt(d, 3), check_less_precise=True)
eq(pd.rolling_quantile(p, 3, 0.5), dd.rolling_quantile(d, 3, 0.5))
eq(pd.rolling_apply(p, 3, mad), dd.rolling_apply(d, 3, mad))
with ignoring(ImportError):
eq(pd.rolling_window(p, 3, 'boxcar'),
dd.rolling_window(d, 3, 'boxcar'))
# Test with edge-case window sizes
eq(pd.rolling_sum(p, 0), dd.rolling_sum(d, 0))
eq(pd.rolling_sum(p, 1), dd.rolling_sum(d, 1))
# Test with kwargs
eq(pd.rolling_sum(p, 3, min_periods=3), dd.rolling_sum(d, 3,
min_periods=3))
def r_count_window(df, windows_size):
return rolling_count(df, windows_size)
def count_nans(self, x, n):
return n - pd.rolling_count(x, n)
def func(window):
return lambda df: pd.rolling_sum(df.fillna(0), window) >= x * pd.rolling_count(df, window)
def func(window):
return lambda df: \
(pd.rolling_sum(df.fillna(0), window)/pd.rolling_count(df, window)).\
rank(axis=1, ascending=ascending) <= x
def rolling_smoother(self, data, stype='rolling_mean', win_size=10, win_type='boxcar', center=False, std=0.1,
beta=0.1,
power=1, width=1):
"""
Perform a espanding smooting on the data for a complete help refer to http://pandas.pydata.org/pandas-docs/dev/computation.html
:param data:
:param stype:
:param win_size:
:param win_type:
:param center:
:param std:
:param beta:
:param power:
:param width:
:moothing types:
ROLLING :
rolling_count Number of non-null observations
rolling_sum Sum of values
rolling_mean Mean of values
rolling_median Arithmetic median of values
rolling_min Minimum
rolling_max Maximum
rolling_std Unbiased standard deviation
rolling_var Unbiased variance
rolling_skew Unbiased skewness (3rd moment)
rolling_kurt Unbiased kurtosis (4th moment)
rolling_window Moving window function
window types:
boxcar
triang
blackman
hamming
bartlett
parzen
bohman
blackmanharris
nuttall
barthann
kaiser (needs beta)
gaussian (needs std)
general_gaussian (needs power, width)
slepian (needs width)
"""
if stype == 'count':
newy = pd.rolling_count(data, win_size)
if stype == 'sum':
newy = pd.rolling_sum(data, win_size)
if stype == 'mean':
newy = pd.rolling_mean(data, win_size)
if stype == 'median':
newy = pd.rolling_median(data, win_size)
if stype == 'min':
newy = pd.rolling_min(data, win_size)
if stype == 'max':
newy = pd.rolling_max(data, win_size)
if stype == 'std':
newy = pd.rolling_std(data, win_size)
if stype == 'var':
newy = pd.rolling_var(data, win_size)
if stype == 'skew':
newy = pd.rolling_skew(data, win_size)
if stype == 'kurt':
newy = pd.rolling_kurt(data, win_size)
if stype == 'window':
if win_type == 'kaiser':
newy = pd.rolling_window(data, win_size, win_type, center=center, beta=beta)
if win_type == 'gaussian':
newy = pd.rolling_window(data, win_size, win_type, center=center, std=std)
if win_type == 'general_gaussian':
newy = pd.rolling_window(data, win_size, win_type, center=center, power=power, width=width)
else:
newy = pd.rolling_window(data, win_size, win_type, center=center)
return newy
def func(window):
return lambda df: pd.rolling_count(df, window) >= x * window * 0.01
def ts_countFn(arr, max_periods):
if not (max_periods): max_periods = len(arr)
return pd.rolling_count(arr, max_periods)
def evaluate(self, table):
expr = self.expr
val = None
if expr is not None:
val = expr.evaluate(table)
return pd.rolling_count(val, self.window)
def func(window):
return lambda df: \
(pd.rolling_sum(df.fillna(0), window)/pd.rolling_count(df, window)).\
rank(axis=1, ascending=ascending) <= x
def func(window):
return lambda df: pd.rolling_count(df, window) >= x * window * 0.01
def func(window):
return lambda df: \
(pd.rolling_sum(df.fillna(0), window)/pd.rolling_count(df, window)).\
rank(axis=1, ascending=ascending).\
div(df.fillna(method='ffill', limit=window).count(axis=1), axis=0) <= x * 0.01
def rolling_count(self, *args, **kwargs):
return MySeries(pd.rolling_count(self.x, *args, **kwargs))
def count_nans(self, x, n):
return n - pd.rolling_count(x, n)
def make_busd_column(dataframe,mins_prior):
column_name = '%s_min_busd' % mins_prior
successfully_made_column = False
while not successfully_made_column:
dataframe[column_name] = pd.rolling_sum(dataframe.busd,mins_prior, freq='T', min_periods = 1) / pd.rolling_count(dataframe.busd,mins_prior, freq='T')
successfully_made_column = assess_column(dataframe, column_name)
def func(window):
return lambda df: pd.rolling_sum(df.fillna(0), window
) <= x * pd.rolling_count(df, window)
def i_counter(grouped):
se = grouped.set_index('INSPECTION_BEGIN_DT')['EVENT_NO']
# se is the time series of inspection dates restricted to a single MINE_ID
se = se.resample("D")
df = pd.DataFrame({'event_no':se, 'i90':pd.rolling_count(se, 90), 'i30':pd.rolling_count(se, 30)})
return df
def func(window):
return lambda df: \
(pd.rolling_sum(df.fillna(0), window)/pd.rolling_count(df, window)).\
rank(axis=1, ascending=ascending).\
div(df.fillna(method='ffill', limit=window).count(axis=1), axis=0) <= x * 0.01