def clean_outliers_moving_average(data_series, length, limit): """ Get a moving average of the ``data_series`` over ``length`` entries, by means of :func:`outlier filter <omc3.utils.outliers.get_filter_mask>`. The values are shifted, so that the averaged value takes ceil((length-1)/2) values previous and floor((length-1)/2) following values into account. Args: data_series: `Series` of data. length: length of the averaging window. limit: points beyond that limit are always filtered. """ LOG.debug( "Filtering and calculating moving average of length {:d}.".format( length)) init_mask = ~data_series.isna() mask = init_mask.copy() for i in range(len(data_series) - length): mask[i:i + length] &= get_filter_mask(data_series[i:i + length], limit=limit, mask=init_mask[i:i + length]) _is_almost_empty_mask(mask, length) data_mav, std_mav = _get_interpolated_moving_average( data_series, ~mask, length) return data_mav, std_mav, mask
def clean_by_tune(tunes: pd.Series, tune_clean_limit): """ Looks for outliers in the tunes pandas Series and returns their indices. Args: tunes (pd.Series): Pandas series with the tunes per BPM and the BPM names as index. tune_clean_limit: No BPM will find as outlier if its distance to the average is lower than this limit. """ bad_bpms_mask = outliers.get_filter_mask(tunes, limit=tune_clean_limit) bad_bpms_names = tunes[~bad_bpms_mask].index return bad_bpms_names
def _filter_by_column(df: pd.DataFrame, column: str, limit: Number) -> pd.DataFrame: """Get the dataframe with all rows dropped filtered by the given column.""" if column not in df.columns: LOG.info(f"{column} not in current file. Skipping cleaning.") return df good_bpms = get_filter_mask(data=df[column], limit=limit) n_good, n_total = sum(good_bpms), len(good_bpms) LOG.info( f"Cleaned {n_total-n_good:d} of {n_total:d} elements in {column} ({n_good:d} remaining)." ) return df.loc[good_bpms, :]
def clean_data(magnet_df, no_autoclean): if no_autoclean: LOG.info( 'Manual cleaning is not yet implemented, no cleaning was performed' ) for plane in PLANES: magnet_df[f"{CLEANED}{plane}"] = True else: LOG.debug('Automatic Tune cleaning') for plane in PLANES: magnet_df[f"{CLEANED}{plane}"] = outliers.get_filter_mask( magnet_df[f"{TUNE}{plane}"].values, x_data=magnet_df[K].values, limit=1e-5) return magnet_df