def detrend(data, how='linear', rule=None, notation=None, suffix=None, units=None, inplace=True, ignore=[], **kwargs):
"""
Returns the detrended fluctuations of a given dataset
Parameters
----------
data: pandas.DataFrame, pandas.Series
dataset to be detrended
how: string
how of average to apply. Currently {'movingmean', 'movingmedian', 'block', 'linear', 'poly'}.
rule: pandas offset string
the blocks for which the trends should be calculated in the block and linear type
window: pandas date offset string or int
if moving mean/median is chosen, this tells us the window size to pass to pandas. If int,
this is the number of points used in the window. If string we will to guess the number of
points from the index.
Small windows (equivalent to 1min approx) work better when using rollingmedian.
block_func: str, function
how to resample in block type. Default is mean but it can be any numpy function
that returns a float. E.g, median.
degree: int
degree of polynomial fit (only if how=='linear' or how=='polynomial')
Returns
-------
pandas.DataFrame or pandas.Series
fluctuations of the input data
"""
from scipy import signal
from . import algs
import pandas as pd
how=algs.stripDown(how.lower(), args='-_')
df=data.copy()
if ignore:
df = df.drop(ignore, axis=1)
defs = algs.get_notation(notation)
#-----------
# We can only use scipy's detrend function safely if index in not datetime
if isinstance(df.index, pd.DatetimeIndex):
df = df - trend(df, how=how, rule=rule, **kwargs)
else:
#-----------
# If possible, try to use scipy's optimized functions
if how=='linear' and rule==None:
try:
df = df.apply(signal.detrend, axis=0, type=how)
except:
df = df - trend(df, how=how, rule=rule, **kwargs)
elif (any(w==how for w in ['block', 'blockaverage', 'blockmean'])) and (rule==None):
try:
df = df.apply(signal.detrend, axis=0, type='constant')
except:
df=df-trend(df, how=how, rule=rule, **kwargs)
#-----------
#-----------
# If not, use our trending function
else:
df = df - trend(df, how=how, rule=rule, **kwargs)
#-----------
#-----------
#-----------
# We rename the columns names to indicate that they are fluctuations
if units:
newunits = { defs.fluctuations % el : units[el] if el in units.keys() else None for el in df.columns }
#-----------
#-----------
# Rename the columns
if suffix != '':
df = df.rename(columns = lambda x: defs.fluctuations % x)
#-----------
#-----------
# If units are not be changed in place, we copy them
if units:
if not inplace:
units = units.copy()
units.update(newunits)
#-----------
if inplace:
return df
else:
return df, units
def trend(data, how='linear', rule=None, window=1200, block_func='mean', center=True, **kwargs):
"""
Wrapper to return the trend given data. Can be achieved using a moving avg, block avg or polynomial fitting
Parameters
----------
data: pandas.DataFrame or pandas.Series
the data whose trend wee seek.
how: string
how of average to apply. Currently {'movingmean', 'movingmedian', 'block', 'linear'}.
rule: string
pandas offset string to define the block in the block average. Default is "10min".
window: pandas date offset string or int
if moving mean/median is chosen, this tells us the window size to pass to pandas. If int,
this is the number of points used in the window. If string we will to guess the number of
points from the index.
Small windows (equivalent to 1min approx) work better when using rollingmedian.
block_func: str, function
how to resample in block type. Default is mean but it can be any numpy function
that returns a float. E.g, median.
degree: int
degree of polynomial fit (only if how=='linear' or how=='polynomial')
Returns
-------
pandas.DataFrame or pandas.Series
trends of data input
"""
import pandas as pd
import algs
import numpy as np
how=algs.stripDown(how.lower(), args='-_')
if ('moving' in how) or ('rolling' in how):
if isinstance(window, str):
window = int(len(data)/len(data.resample(window)))
elif isinstance(window, int):
pass
else:
raise TypeError('Window of moving function should be either an int or a pandas datetime offset string.')
#-------
# Performs moving average on the data with window
if ('mean' in how) or ('average' in how):
return pd.rolling_mean(data, window=window, center=center, **kwargs)
#-------
#-------
# Performs moving median on the data with window
elif 'median' in how:
return pd.rolling_median(data, window=window, center=center, **kwargs)
#-------
#-------
# In case how not found.
else:
raise KeyError('Method of trending not found. Check how keyword options with help(trend).')
#-------
elif any(w==how for w in ['block', 'blockaverage', 'blockmean']):
#-------
# performs block average on the data with the window being the rule. Assumes that frequency is constant
if rule==None:
if isinstance(data, pd.DataFrame):
return data.apply(lambda x: [np.mean(x)]*len(x), axis=0)
elif isinstance(data, pd.Series):
return data.apply(lambda x: np.mean(x))
else:
freq=data.index.inferred_freq
aux = data.resample(rule, how=block_func, **kwargs)
aux.loc[ data.index[-1] ] = np.nan
return aux.resample(freq, fill_method='pad')
#-------
elif any(w in how for w in ['linear', 'polynomial', 'poly']):
#-------
# performs a polynomial fit on the data in blocks of "rule"
return data.polyfit(rule=rule, **kwargs)
#-------
else:
#-------
# if no how can be identified
raise KeyError('Method of trending not found. Check how keyword options with help(trend).')