def check_std(data, tables, detrend=False, detrend_kw={'how':'linear'}, chunk_size='2min', falseverbose=False):
"""
Checks dataframe for columns with too small of a standard deviation
Parameters
----------
data: pandas.DataFrame
dataset whose standard deviation to check
tables: pandas.DataFrame
dataset containing the standard deviation limits for each column
detrend: bool
whether to work with the absolute series and the fluctuations
detrend_kw: dict
keywords to pass to pymicra.detrend with detrend==True
chunk_size: str
pandas datetime offset string
Returns
-------
valid: pandas.Series
contatining True of False for each column. True means passed the test.
"""
import signal as pmdata
import numpy as np
import pandas as pd
from . import algs
#-----------
# Detrend the data or not
if detrend:
df = pmdata.detrend(data, suffix='', **detrend_kw)
else:
df = data.copy()
#-----------
#-----------
# Separate into smaller bits or just get the full standard deviation
if chunk_size:
stds_list = algs.resample(df, chunk_size, how=np.std).dropna()
else:
stds_list = pd.DataFrame(index=[df.index[0]], columns = df.columns)
stds_list.iloc[0, :] = df.apply(np.std)
#-----------
#-----------
# Check each chunk separately
validcols = ~( stds_list < tables.loc['std_limits'] )
if falseverbose and (False in validcols.values):
falsecols = [ el for el in df.columns if False in validcols.loc[:, el].values ]
print('STD test: failed variables and times are\n{0}\n'.format(validcols.loc[:, falsecols]))
#-----------
valid = validcols.all(axis=0)
return valid
def check_RA(data, detrend=True, detrend_kw={'how':'linear'},
RAT_vars=None, RAT_points=50, RAT_significance=0.05):
"""
Performs the Reverse Arrangement Test in each column of data
Parameters
----------
data: pandas.DataFrame
to apply RAT to each column
detrend_kw: dict
keywords to pass to pymicra.detrend
RAT_vars:
list of variables to which apply the RAT
RAT_points: int
if it's an int N, then reduce each column to N points by averaging. If None,
then the whole columns are used
RAT_significance: float
significance with which to apply the RAT
Returns
-------
valid: pd.Series
True or False for each column. If True, column passed the test
"""
import signal as pmdata
#-----------
# Detrend the data
if detrend:
df = pmdata.detrend(data, suffix='', **detrend_kw)
else:
df = data.copy()
#-----------
#-----------
# If RAT_vars is given, apply reverse arrangement only on these variables
if RAT_vars:
valid = df[RAT_vars].apply(pmdata.reverse_arrangement, axis=0, points_number=RAT_points, alpha=RAT_significance)
elif RAT_vars==None:
valid = df.apply(pmdata.reverse_arrangement, axis=0, points_number=RAT_points, alpha=RAT_significance)
else:
raise TypeError('Check RAT_vars keyword')
#-----------
return valid