def text_analysis(dbHost="", dbPort="", userName="", password="", dbName="", query="", textColumns="", parametersObj="", columnsArray=""): ############## # connecting to BD ############## data_text = pd.read_csv( "E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/googleplaystore_user_reviews.csv" ) data, columnsArray_e = columns_data_type(data_text, columnsArray) data = data_text[textColumns].dropna() corpus_bow, corpus_tfidf, id2word = clean_text(data) key_words, doc_topic = lda_model(corpus=corpus_bow, id2word=id2word, num_topics=5, num_term=10) sentiment = sentiment_analysis(data_text=data_text, data=data, doc_topic=doc_topic, sent_col=parametersObj) return [sentiment, doc_topic, key_words]
def clustering(dbHost="", dbPort="", userName="", password="", dbName="", query="", xValues="", parametersObj="", columnsArray=""): ############## # connecting to BD ############## # data = pd.read_csv("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/50000_Sales_Records_Dataset_e.csv") # data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/quine.xlsx") # data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/airfares.xlsx") data = pd.read_csv( "E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/AAPL.csv") data, columnsArray_e = columns_data_type(data[0:100], columnsArray) ent_cor, chisq_dependency, data, rm_cols, miss_cols, obj_t = correlations( data, columnsArray=columnsArray_e, method='predict', no_rem_col='none') print('Data info before clustering') print(data.info()) output = kmeans_model(data[xValues], xValues) return output
def forecasting(dbHost="",dbPort="",userName="",password="",dbName="",query="",xValues="",yValue="",parametersObj="",columnsArray=""): ############## # connecting to BD ############## # data = pd.read_csv("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/50000_Sales_Records_Dataset_e.csv") # data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/quine.xlsx") # data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/airfares.xlsx") data = pd.read_csv("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/AAPL.csv") data, columnsArray_e = columns_data_type(data[0:100], columnsArray) ent_cor,chisq_dependency,data,rm_cols, miss_cols, obj_t = correlations(data, columnsArray=columnsArray_e, method = 'impute', no_rem_col =['date']) output = forecasting_model(data,variable_col = xValues,date_col = yValue, model = parametersObj, independentVariables='',test_split = 0.4) return output
def decisiontree(dbHost="",dbPort="",userName="",password="",dbName="",query="",yValue="",xValues="",parametersObj="",columnsArray=""): ############## # connecting to BD ############## # data = pd.read_csv("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/50000_Sales_Records_Dataset_e.csv") # data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/quine.xlsx") data = pd.read_excel("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/airfares.xlsx") # data = pd.read_csv("E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/AAPL.csv") data, columnsArray_e = columns_data_type(data[0:100], columnsArray) ent_cor,chisq_dependency,data,rm_cols, miss_cols, obj_t = correlations(data, columnsArray=columnsArray_e, method='predict', no_rem_col='none') feature = xValues feature.append(yValue) df = data[feature] hf = h2o.H2OFrame(df) train, valid, test = hf.split_frame(ratios=[.8, .1]) # GBM model gbm = H2OGradientBoostingEstimator() gbm.train(xValues, yValue, training_frame= train, validation_frame=valid) var_imp = pd.DataFrame(gbm.varimp()) var_imp.columns = ['variable','relative_importance','scaled_importance','percentage'] # metrics variable_importance = var_imp[var_imp['relative_importance']>=0.1] variable_importance = variable_importance['variable'] gbm_cm = gbm.confusion_matrix() gbm_cm = gbm_cm.table.as_data_frame() gbm_prec = gbm.precision()[0][0] gbm_f1 = gbm.F1()[0][0] gbm_acc = gbm.accuracy()[0][0] gbm_rec = gbm_f1*gbm_prec/(2*gbm_prec-gbm_f1) # Tree info from h2o.tree import H2OTree tree = H2OTree(model = gbm, tree_number = 1, tree_class = None) nlg_tree = tree.descriptions tree_json = tree2json(tree) output = [tree_json, gbm_prec, gbm_rec, gbm_acc, gbm_cm, variable_importance, nlg_tree] return output
def linear_regression(dbHost="", dbPort="", userName="", password="", dbName="", query="", yValue="", xValues="", parametersObj="", columnsArray=""): ############## # connecting to BD ############## data = pd.read_csv( "E:/Work/aspirantura/kafedra/upwork/RtoP/Models_sep/Datasets/50000_Sales_Records_Dataset_e.csv" ) data, columnsArray_e = columns_data_type(data[0:100], columnsArray) ent_cor, chisq_dependency, data, rm_cols, miss_cols, obj_t = correlations( data, columnsArray=columnsArray_e, method='predict') #print(ent_cor,chisq_dependency, rm_cols, miss_cols) num_data = data.select_dtypes(include=['number']).copy() variable_imp = variable_importance_h2o(data, list(num_data.columns), yValue) vif_var = vif(num_data, yValue, 10) #print(variable_imp, vif_var) num_data_inv = transformation_inv(num_data, obj_t) x_mean = num_data_inv[xValues].mean() x_min = num_data_inv[xValues].min() x_max = num_data_inv[xValues].max() x_quant25 = num_data_inv[xValues].quantile(0.25) x_quant50 = num_data_inv[xValues].quantile(0.5) x_quant75 = num_data_inv[xValues].quantile(0.75) x_skew = scipy.stats.skew(num_data_inv[xValues]) if (data[yValue].dtypes == 'float') or (data[yValue].dtypes == 'int'): print( "Finding variable importance by taking given numeric variable as a dependent variable" ) hf = h2o.H2OFrame(num_data) #hf.col_names train, valid, test = hf.split_frame(ratios=[.8, .1]) glm_model = H2OGeneralizedLinearEstimator(family='gaussian') glm_model.train(xValues, yValue, training_frame=train, validation_frame=valid) print(glm_model) predicted = glm_model.predict(test_data=test) test_inv = transformation_inv(test.as_data_frame(), obj_t) true_y = test_inv[yValue] test[yValue] = predicted test_inv = transformation_inv(test.as_data_frame(), obj_t) pred_y = test_inv[yValue] linear_regr = [ pred_y, true_y, glm_model.r2(), glm_model.rmse(), ent_cor, glm_model.coef(), variable_imp, vif_var, [x_mean, x_min, x_max, x_quant25, x_quant50, x_quant75, x_skew] ] else: print( "Finding variable importance by taking categorical variables as dependent variable" ) gbm = H2OGradientBoostingEstimator() gbm.train(xValues, yValue, training_frame=train, validation_frame=valid) # print(gbm) predicted2 = gbm.predict(test_data=test) test_inv = transformation_inv(test.as_data_frame(), obj_t) true_y = test_inv[yValue] test[yValue] = predicted2 test_inv = transformation_inv(test.as_data_frame(), obj_t) pred2_y = test_inv[yValue] linear_regr = [ pred2_y, true_y, glm_model.r2(), glm_model.rmse(), ent_cor, glm_model.coef(), variable_imp, vif_var ] return linear_regr