def outlier_detection(x_raw, y_raw): """ Filter all ourlier points :param x_raw: feature in ndarray :param y_raw: label in ndarray :return x_clean, y_clean: cleaned feature and label in ndarray """ # TODO Filter the outliers. print() print("Detecting outliers...") print("Before outlier detection: {}".format(x_raw.shape)) outliers_fraction = 0.04 random_state = np.random.RandomState(42) # all outlier detection method candidate list as follows classifiers = { 'Angle-based Outlier Detector (ABOD)': ABOD(contamination=outliers_fraction), 'Cluster-based Local Outlier Factor': CBLOF(contamination=outliers_fraction, check_estimator=False, random_state=random_state), 'Feature Bagging': FeatureBagging(contamination=outliers_fraction, random_state=random_state), 'Histogram-base Outlier Detection (HBOS)': HBOS(contamination=outliers_fraction), 'Isolation Forest': IForest(contamination=outliers_fraction, random_state=random_state), 'K Nearest Neighbors (KNN)': KNN(contamination=outliers_fraction), 'Local Outlier Factor (LOF)': LOF(contamination=outliers_fraction), 'Minimum Covariance Determinant (MCD)': MCD(contamination=outliers_fraction, random_state=random_state), 'One-class SVM (OCSVM)': OCSVM(contamination=outliers_fraction), 'Principal Component Analysis (PCA)': PCA(contamination=outliers_fraction, random_state=random_state), 'Improving Supervised Outlier Detection with Unsupervised Representation Learning': XGBOD(contamination=outliers_fraction), } clf_name = 'Isolation Forest' clf = IForest(contamination=outliers_fraction, random_state=random_state) # clf_name = 'Angle-based Outlier Detector (ABOD)' # clf = ABOD(contamination=outliers_fraction, method='default') clf.fit(x_raw) y_pred = clf.predict(x_raw) # for pyod, 1 means outliers and 0 means inliers # for sklearn, -1 means outliers and 1 means inliers idx_y_pred = [i for i in range(0, 1212) if y_pred[i] == 1] x_clean = del_rowsorcolumns(x_raw, idx_y_pred, axis=0) y_clean = del_rowsorcolumns(y_raw, idx_y_pred, axis=0) print("After outlier detection: {}".format(x_clean.shape)) assert (x_clean.shape[0] == y_clean.shape[0]) return x_clean, y_clean
def define_classifiers(random_state, outliers_fraction): classifiers = { 'Angle-based Outlier Detector (ABOD)': ABOD(contamination=outliers_fraction), 'Cluster-based Local Outlier Factor': CBLOF(contamination=outliers_fraction, check_estimator=False, random_state=random_state), 'Feature Bagging': FeatureBagging(contamination=outliers_fraction, random_state=random_state), 'Histogram-base Outlier Detection (HBOS)': HBOS(contamination=outliers_fraction), 'Isolation Forest': IForest(contamination=outliers_fraction, random_state=random_state), 'K Nearest Neighbors (KNN)': KNN(contamination=outliers_fraction), 'Local Outlier Factor (LOF)': LOF(contamination=outliers_fraction), 'Minimum Covariance Determinant (MCD)': MCD(contamination=outliers_fraction, random_state=random_state), 'One-class SVM (OCSVM)': OCSVM(contamination=outliers_fraction), 'Principal Component Analysis (PCA)': PCA(contamination=outliers_fraction, random_state=random_state) } return classifiers
def setUp(self): # Define data file and read X and y # Generate some data if the source data is missing this_directory = path.abspath(path.dirname(__file__)) mat_file = 'cardio.mat' try: mat = loadmat(path.join(*[this_directory, 'data', mat_file])) except TypeError: print('{data_file} does not exist. Use generated data'.format( data_file=mat_file)) X, y = generate_data(train_only=True) # load data except IOError: print('{data_file} does not exist. Use generated data'.format( data_file=mat_file)) X, y = generate_data(train_only=True) # load data else: X = mat['X'] y = mat['y'].ravel() X, y = check_X_y(X, y) self.X_train, self.X_test, self.y_train, self.y_test = \ train_test_split(X, y, test_size=0.4, random_state=42) self.base_estimators = [LOF(), LOF(), IForest(), COPOD()] self.clf = SUOD(base_estimators=self.base_estimators) self.clf.fit(self.X_train) self.roc_floor = 0.7
def train_model(X_train, contamination, model): """ Train the model based on the user's choice (KNN, IForest, OCSVM). Parameters ---------- X_train : list of shape (n_train, n_features) containing the training set with only features. contamination : float representing the expected proportion of anomalies in the training set. model : string, claiming the model to use for evaluating the confidence. It can be one of: KNN, IForest, OCSVM. Returns ---------- clf : obj with the model trained on the training set. """ np.random.seed(331) n = np.shape(X_train)[0] if model == 'KNN': clf = KNN(n_neighbors=max(np.int(n*contamination),1), contamination = contamination).fit(X_train) elif model == 'IForest': clf = IForest(contamination = contamination, random_state = 331).fit(X_train) elif model == 'OCSVM': clf = OCSVM(contamination = contamination).fit(X_train) return clf
def densityBased(self): ''' @brief Function that implements the dependency based component @param self @return It returns the vector with the scores of the instances ''' # Initialize the scores scores = np.array([0] * len(self.dataset)).astype(float) for i in range(self.num_iter): iforest = IForest(contamination=self.contamination, behaviour="new") # Number in the interval [50, 1000] subsample_size = np.random.randint(50, 1001) sample = [] if subsample_size >= len(self.dataset): sample = list(range(len(self.dataset))) else: # Take the sample and train the model sample = np.random.choice(len(self.dataset), size=subsample_size, replace=False) iforest.fit(self.dataset[sample]) # Update the score to compute the mean scores[sample] += iforest.decision_scores_ # Return the mean scores = scores / self.num_iter scores = scale(scores) return scores
def construct_raw_base_estimators(): from pyod.models.knn import KNN from pyod.models.lof import LOF from pyod.models.cblof import CBLOF from pyod.models.hbos import HBOS from pyod.models.iforest import IForest from pyod.models.abod import ABOD from pyod.models.ocsvm import OCSVM estimator_list = [] # predefined range of n_neighbors for KNN, AvgKNN, and LOF k_range = [3, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] for k in k_range: estimator_list.append( KNN(n_neighbors=k, method="largest", contamination=0.05)) estimator_list.append( KNN(n_neighbors=k, method="mean", contamination=0.05)) estimator_list.append(LOF(n_neighbors=k, contamination=0.05)) # predefined range of nu for one-class svm nu_range = [0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99] for nu in nu_range: estimator_list.append(OCSVM(nu=nu, contamination=0.05)) # predefined range for number of estimators in isolation forests n_range = [10, 20, 50, 70, 100, 150, 200, 250] for n in n_range: estimator_list.append( IForest(n_estimators=n, random_state=42, contamination=0.05)) return estimator_list
def train_model(X, Y, contamination, name, from_scratch=True): model_dir = './model' if not os.path.exists(model_dir): os.mkdir(model_dir) file_name = name + '.pkl' if from_scratch: if name == 'ocsvm': model = OCSVM(contamination=contamination) model.fit(X) elif name == 'iforest': model = IForest(contamination=contamination) model.fit(X) elif name == 'lof': model = LOF(contamination=contamination) model.fit(X) elif name == 'knn': model = KNN(contamination=contamination) model.fit(X) elif name == 'xgbod': model = XGBOD(contamination=contamination) model.fit(X, Y) save(model, model_dir, file_name) else: model = load(model_dir, file_name) return model
def train(): # initiate the train try: # 1-. Take data and configuration data = pd.read_csv(training_path, index_col=0) # Read in any configuration stored with open(param_path, 'r') as tc: hyper_parameters = json.load(tc) # 2-. Set up # instantiate the Isolation Forest model model = IForest(contamination=hyper_parameters['contamination'], behaviour='new') model.fit(data) # fit # 3-. Save the model model_name = 'great_model' with open(os.path.join(model_path, '{}.pkl'.format(model_name)), 'wb') as out: pickle.dump(model, out, protocol=0) # consider that the train fails except Exception as e: # write the log trc = traceback.format_exc() with open(os.path.join(output_path, 'failure'), 'w') as s: s.write('Exception during train: ' + str(e) + '\n' + trc) sys.exit(255)
def model_init(self, model): """Model initialisation of a single model. """ if self.model == 'pca': self.models[model] = PCA(contamination=self.contamination) elif self.model == 'loda': self.models[model] = LODA(contamination=self.contamination) elif self.model == 'iforest': self.models[model] = IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination) elif self.model == 'cblof': self.models[model] = CBLOF(n_clusters=3, contamination=self.contamination) elif self.model == 'feature_bagging': self.models[model] = FeatureBagging( base_estimator=PCA(contamination=self.contamination), contamination=self.contamination) elif self.model == 'copod': self.models[model] = COPOD(contamination=self.contamination) elif self.model == 'hbos': self.models[model] = HBOS(contamination=self.contamination) else: self.models[model] = HBOS(contamination=self.contamination) self.custom_model_scalers[model] = MinMaxScaler()
def load_classifiers(outliers_fraction): outliers_fraction = min(0.5, outliers_fraction) random_state = np.random.RandomState(42) # Define nine outlier detection tools to be compared classifiers = { 'Angle-based Outlier Detector (ABOD)': ABOD(contamination=outliers_fraction), 'Cluster-based Local Outlier Factor (CBLOF)': CBLOF(contamination=outliers_fraction, check_estimator=False, random_state=random_state), 'Feature Bagging': FeatureBagging(LOF(n_neighbors=35), contamination=outliers_fraction, random_state=random_state), 'Histogram-base Outlier Detection (HBOS)': HBOS(contamination=outliers_fraction), 'Isolation Forest': IForest(contamination=outliers_fraction, random_state=random_state, behaviour="new"), 'K Nearest Neighbors (KNN)': KNN(contamination=outliers_fraction), 'Average KNN': KNN(method='mean', contamination=outliers_fraction), 'Local Outlier Factor (LOF)': LOF(n_neighbors=35, contamination=outliers_fraction), 'Minimum Covariance Determinant (MCD)': MCD(contamination=outliers_fraction, random_state=random_state), 'One-class SVM (OCSVM)': OCSVM(contamination=outliers_fraction), 'Principal Component Analysis (PCA)': PCA(contamination=outliers_fraction, random_state=random_state) } return classifiers
def anomaly_detection(data, label): X = data[data.select_dtypes('number').columns.tolist()] y = data[label] y = y.values X = X.drop([label], axis=1) sc = StandardScaler() X = pd.DataFrame(data=sc.fit_transform(X), columns=X.columns) ifo = IForest(contamination=0.01, behaviour='new', n_estimators=1000, max_samples=1024, n_jobs=-1, verbose=1) ifo.fit(X) ifo_pred = ifo.labels_ print('ROC score for Isolation forest: ', roc_auc_score(y, ifo_pred)) utilities.plot_outlier_scores( y, ifo.decision_scores_, bw=0.1, title='Fraud, Isolation forest. (n_estimators={})'.format( ifo.n_estimators)) ae = AutoEncoder(hidden_neurons=[25, 20, 15, 20, 25], hidden_activation='relu', output_activation='sigmoid', optimizer='adam', epochs=20, batch_size=128, dropout_rate=0.2, l2_regularizer=0.0, validation_size=0.1, preprocessing=False, verbose=1, random_state=1, contamination=0.01) ae.fit(X) ae_pred = ae.labels_ print('ROC score for Autoencoder: ', roc_auc_score(y, ae_pred)) utilities.plot_outlier_scores( y, ae.decision_scores_, bw=0.1, title='Fraud, Autoencoder. (epochs={})'.format(ae.epochs)) # Too long to train, under-sample needed lof = LOF(n_neighbors=int(y.sum() * 1.3), contamination=0.01, n_jobs=-1) lof.fit(X) lof_pred = lof.labels_ print('ROC score for LOF: ', roc_auc_score(y, lof_pred)) utilities.plot_outlier_scores( y, lof.decision_scores_, bw=0.1, title='Fraud, Local outliers factor. (n_neighbors={})'.format( lof.n_neighbors)) return y, ifo_pred, ae_pred, lof_pred
def __load_classifiers(self): outliers_fraction = 0.05 random_state = np.random.RandomState(0) classifiers = { 'Cluster-based Local Outlier Factor (CBLOF)': CBLOF(contamination=outliers_fraction, check_estimator=False, random_state=random_state), 'Feature Bagging': FeatureBagging(LOF(n_neighbors=35), contamination=outliers_fraction, random_state=random_state), 'Histogram-base Outlier Detection (HBOS)': HBOS(contamination=outliers_fraction), 'Isolation Forest': IForest(contamination=outliers_fraction, random_state=random_state), 'K Nearest Neighbors (KNN)': KNN(contamination=outliers_fraction), 'Average KNN': KNN(method='mean', contamination=outliers_fraction), 'Local Outlier Factor (LOF)': LOF(n_neighbors=35, contamination=outliers_fraction), 'Minimum Covariance Determinant (MCD)': MCD(contamination=outliers_fraction, random_state=random_state), 'One-class SVM (OCSVM)': OCSVM(contamination=outliers_fraction), } return classifiers
def fit(self, X_train, y_train=None): """Fit the model. y is ignored in unsupervised methods. Parameters ---------- X_train : numpy array of shape (n_samples, n_features) The input samples. y_train : Ignored Not used, present for API consistency by convention. Returns ------- self : object The fitted estimator. """ self.model_ = IForest( n_estimators=self.n_estimators, max_samples=self.max_samples, contamination=self.contamination, max_features=1., bootstrap=False, n_jobs=-1, behaviour='deprecated', # no use any more in sklean 0.24. random_state=self.random_state, verbose=self.verbose) self.model_.fit(X=X_train) return self
def dorc(preprocessedData, random_state, outliers_fraction=0.1): t0 = time.time() clf = IForest(contamination=outliers_fraction, random_state=random_state, n_jobs=-1) clf.fit(preprocessedData) scores = clf.decision_function(preprocessedData) # Apply IQR-based criteria to identify rare cells for further downstream analysis. q3 = np.percentile(scores, 75) iqr = stats.iqr(scores) th = q3 + (1.5 * iqr) # Select indexes that satisfy IQR-based thresholding criteria. indIqr = np.where(scores >= th)[0] print('shape of selected cells : {}'.format(indIqr.shape)) # Create a file with binary predictions predictions = np.zeros(preprocessedData.shape[0]) predictions[indIqr] = 1 # Replace predictions for rare cells with '1'. t1 = time.time() duration = round(t1 - t0, ndigits=4) print("Total running DoRC time is :" + str(duration) + " s") return predictions, scores, duration
def calculate(method, total_roc, total_prn, x_train, x_test, y_train, y_test): if method == 'KNN': clf = KNN() elif method == 'CBLOF': clf = CBLOF() elif method == 'PCA': clf = PCA() else: clf = IForest() clf.fit(x_train) # 使用x_train训练检测器clf # 返回训练数据x_train上的异常标签和异常分值 y_train_pred = clf.labels_ # 返回训练数据上的分类标签 (0: 正常值, 1: 异常值) y_train_scores = clf.decision_scores_ # 返回训练数据上的异常值 (分值越大越异常) print("On train Data:") evaluate_print(method, y_train, y_train_scores) # 用训练好的clf来预测未知数据中的异常值 y_test_pred = clf.predict(x_test) # 返回未知数据上的分类标签 (0: 正常值, 1: 异常值) y_test_scores = clf.decision_function(x_test) # 返回未知数据上的异常值 (分值越大越异常) print("On Test Data:") evaluate_print(method, y_test, y_test_scores) y_true = column_or_1d(y_test) y_pred = column_or_1d(y_test_scores) check_consistent_length(y_true, y_pred) roc = np.round(roc_auc_score(y_true, y_pred), decimals=4), prn = np.round(precision_n_scores(y_true, y_pred), decimals=4) total_roc.append(roc) total_prn.append(prn)
def __init__(self, get_top=0.8, if_params={}, knn_params={}): super(IForestSupervisedKNN, self).__init__() self.get_top = get_top self.is_fitted = False self.iforest = IForest(**if_params) self.knn = KNN(**knn_params)
def iforest(X_train, X_test, Y_train, Y_test): from pyod.models.iforest import IForest model = IForest(random_state=0) model.fit(X_train) pred = model.predict(X_test) acc = np.sum(pred == Y_test) / X_test.shape[0] print(acc) return (acc * 100)
def do_iforest(x, n_estimators=100, max_samples=512): clf = IForest(behaviour="new", n_estimators=n_estimators, max_samples=max_samples, random_state=None) y_pred = clf.fit_predict(x) scores = clf.decision_function(x) index = np.where(y_pred == 1)[0] return clf, scores, index
def remove_outliars(dft, target_col): ol_model = IForest() #### can be used as a hyperparameter ol_model.fit(dft.drop(columns=target_col)) dft['is_outliar'] = ol_model.labels_ dft = dft[dft['is_outliar'] != 1] dft = dft.drop(columns='is_outliar') print("Completed Outliar Detection - ", datetime.datetime.now()) return dft
def outlier_iforest(data, **kwargs): import pandas as pd from pyod.models.iforest import IForest contamination = float(kwargs.pop('contamination')) clf = IForest(contamination=contamination) clf.fit(data) pred = clf.labels_ df = pd.DataFrame(pred, columns=['is_outlier']) ret = pd.concat([data, df], axis=1) return ret
def S2(self): self.S1() water_data = self.water_data result = self.result # 数据预处理及模型训练 clean_data = water_data[water_data['S1'] == 0] Y = pd.DataFrame(index=clean_data.index, columns=['S2']) X_train = np.array(clean_data.iloc[:, 1:12]) name = list(clean_data.iloc[:, 1:12].columns.values) scaler = preprocessing.StandardScaler().fit(X_train) X_train = scaler.transform(X_train) clf1 = IForest(contamination=0.05, max_features=11, bootstrap=True) clf2 = KNN(contamination=0.05, n_neighbors=100) clf3 = HBOS(contamination=0.05, n_bins=10) clf4 = PCA(contamination=0.05) clf1.fit(X_train) clf2.fit(X_train) clf3.fit(X_train) clf4.fit(X_train) Y['S2'] = clf1.labels_ * clf2.labels_ * clf3.labels_ * clf4.labels_ water_data = pd.concat([water_data, Y], axis=1) # water_data.loc[water_data['S2'].isna(),['S2']]=0,将S1中异常的,在S2中标注为0; result['统计异常'] = water_data['S2'].values # 寻找异常维度 from sklearn.neighbors import KernelDensity clean_data = water_data[water_data['S1'] == 0] dens = pd.DataFrame(index=clean_data.index, columns=[ 'temperature', 'pH', 'EC', 'ORP', 'DO', 'turbidity', 'transparency', 'COD', 'P', 'NH3N', 'flux' ]) for i in dens.columns: kde = KernelDensity(kernel='gaussian', bandwidth=0.5).fit( clean_data[i].values.reshape(-1, 1)) dens[i] = np.exp( kde.score_samples(clean_data[i].values.reshape(-1, 1))) dens = dens.iloc[:, 0:11].rank() dens['S2_names'] = dens.idxmin(axis=1) water_data = pd.concat([water_data, dens['S2_names']], axis=1) self.water_data = water_data result['统计异常维度'] = water_data['S2_names'].values # 存储模型 joblib.dump(scaler, "./water_model/S2_scaler") joblib.dump(clf1, "./water_model/S2_Iforest")
def OD_detect(df, id_col=None, contamination=0.05, trans_cols=None): """ use pyod lib to find 5% outlier in dataset """ df = df.copy() OD_clfs = { "HBOS": HBOS(contamination=contamination), "IForest": IForest(contamination=contamination), "CBLOF": CBLOF(contamination=contamination, n_clusters=5), # "OCSVM": OCSVM(contamination=contamination), "PCA": PCA(contamination=contamination) } results_list = [] od_cols = ["id", "name", "result", "label"] if id_col is None: s_id = df.index od_cols = df.columns else: s_id = df[id_col] X_cols = df.columns.drop(id_col) if trans_cols is not None: for col in trans_cols: df[col] = PowerTransformer().fit_transform(df[col].values.reshape( -1, 1)) for clf_name, clf in OD_clfs.items(): od_result = pd.DataFrame(columns=od_cols) # create an empty dataframe od_result["id"] = s_id od_result['name'] = clf_name print(f"{clf_name}, {clf}") clf.fit(df[X_cols]) od_result['result'] = clf.decision_scores_ od_result['label'] = clf.labels_ results_list.append(od_result) od_results_df = pd.concat(results_list, axis=0, ignore_index=True) job_name = f'{pd.datetime.now():%H%M}' od_results_df['job_name'] = job_name od_results_df.to_sql('t_ml', engine, if_exists='append', schema='wh_v1', method=psql_insert_copy) print( f"OD results {od_results_df.shape}exported to database{engine},job_name={job_name}" ) return od_results_df
def setUp(self): self.n_train = 200 self.n_test = 100 self.contamination = 0.1 self.roc_floor = 0.8 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, contamination=self.contamination, random_state=42) self.clf = IForest(contamination=self.contamination, random_state=42) self.clf.fit(self.X_train)
def models_init(self): """Models initialisation. """ self.model = self.configuration.get('model', 'pca') if self.model == 'pca': self.models = { model: PCA(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'loda': self.models = { model: LODA(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'iforest': self.models = { model: IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'cblof': self.models = { model: CBLOF(n_clusters=3, contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'feature_bagging': self.models = { model: FeatureBagging( base_estimator=PCA(contamination=self.contamination), contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'copod': self.models = { model: COPOD(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'hbos': self.models = { model: HBOS(contamination=self.contamination) for model in self.models_in_scope } else: self.models = { model: HBOS(contamination=self.contamination) for model in self.models_in_scope } self.custom_model_scalers = { model: MinMaxScaler() for model in self.models_in_scope }
def setUp(self): self.n_train = 100 self.n_test = 50 self.contamination = 0.1 self.roc_floor = 0.6 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, contamination=self.contamination) self.clf = IForest(contamination=self.contamination) self.clf.fit(self.X_train)
def getOutlierIForest(dataset): ''' @brief Function that executes IForest algorithm on the dataset and obtains the labels of the dataset indicating which instance is an inlier (0) or outlier (1) @param dataset Dataset on which to try the algorithm @return It returns a list of labels 0 means inlier, 1 means outlier ''' # Initializating the model without verbose ifor = IForest(verbose=0) # Fits the data and obtains labels ifor.fit(dataset) # Return labels return ifor.labels_
def __init__(self, nu=0.1): self.base_estimators = [ #OCSVM(contamination=nu), KNN(n_neighbors=100, contamination=nu), KNN(n_neighbors=25, contamination=nu), KNN(n_neighbors=5, contamination=nu), IForest(contamination=nu) ] self.model = SUOD(base_estimators=self.base_estimators, rp_flag_global=False, bps_flag=True, approx_flag_global=False) self.scores = None
def detect(self, X, y=None): """ :param X: Dataframe :param y: np.array :return: outlier scores """ rng = np.random.RandomState(42) # 构造训练样本 n_estimators = 200 # 森林中树的棵数 outliers_fraction = 0.5 # 异常样本比例 clf = IForest(max_samples='auto', random_state=rng, contamination=outliers_fraction, n_estimators=n_estimators) clf.fit(X) scores = clf.decision_function(X) return scores
def detect_outliers(stocks: list, all_stocks_cip: pd.DataFrame, rules=None): """ Returns a dataframe describing those outliers present in stocks based on the provided rules. """ if rules is None: rules = default_point_score_rules() str_rules = { str(r):r for r in rules } rows = [] stocks_by_sector_df = stocks_by_sector() # NB: ETFs in watchlist will have no sector stocks_by_sector_df.index = stocks_by_sector_df['asx_code'] for stock in stocks: #print("Processing stock: ", stock) try: sector = stocks_by_sector_df.at[stock, 'sector_name'] sector_companies = list(stocks_by_sector_df.loc[stocks_by_sector_df['sector_name'] == sector].asx_code) # day_low_high() may raise KeyError when data is currently being fetched, so it appears here... day_low_high_df = day_low_high(stock, all_stocks_cip.columns) except KeyError: warning(None, "Unable to locate watchlist entry: {} - continuing without it".format(stock)) continue state = { 'day_low_high_df': day_low_high_df, # never changes each day, so we init it here 'all_stocks_change_in_percent_df': all_stocks_cip, 'stock': stock, 'daily_range_threshold': 0.20, # 20% at either end of the daily range gets a point } points_by_rule = defaultdict(int) for date in all_stocks_cip.columns: market_avg = all_stocks_cip[date].mean() sector_avg = all_stocks_cip[date].filter(items=sector_companies).mean() stock_move = all_stocks_cip.at[stock, date] state.update({ 'market_avg': market_avg, 'sector_avg': sector_avg, 'stock_move': stock_move, 'date': date }) for rule_name, rule in str_rules.items(): points_by_rule[rule_name] += rule(state) d = { 'stock': stock } d.update(points_by_rule) rows.append(d) df = pd.DataFrame.from_records(rows) df = df.set_index('stock') print(df) from pyod.models.iforest import IForest clf = IForest() clf.fit(df) scores = clf.predict(df) results = [row[0] for row, value in zip(df.iterrows(), scores) if value > 0] #print(results) print("Found {} outlier stocks".format(len(results))) return results
def fit_transform(self, df_train, df_corrupted): pyod_model = IForest(contamination=0.25) df_outliers_num = self.num_out_detect(df_train, df_corrupted, pyod_model) df_outliers_cat = self.cat_out_detect(df_train, df_corrupted) df_outliers = df_outliers_num.join(df_outliers_cat, how='inner') for col in df_corrupted.columns: for i in df_outliers.index: if df_outliers.loc[i, col + "_outlier"] == 1: df_outliers.loc[i, col] = np.nan return df_outliers, self.predictors