def inscrawler_to_file(): print("Starting process...") result = threaded_crawler() print("Inscrawler Done...") total, ignored, processed_result = preprocess(result) print("Processed Results...") print(str(total) + " total results, " + str(ignored) + " ignored results.") generate_folders(processed_result) print("Generated Folders for Influencers...") f = open("user.txt", "w+") f.write(json.dumps(processed_result)) print("Saved Images to txt file") save_images(processed_result) print("Downloaded Images...") print("Done!")
def __init__(self, epochs: int = 500, lr: float = 0.01, dropout_chance: float = 0.5, hiddensize: int = 32): self.epochs = epochs self.lr = lr self.dropout_chance = dropout_chance self.hiddensize = hiddensize self.train_x, self.test_x, self.train_y, self.test_y = preprocess() self.syn0 = 2 * np.random.random((self.hiddensize, 4)) - 1 self.syn1 = 2 * np.random.random((3, self.hiddensize)) - 1 self.b = 1 self.losses = []
def open_and_extract(): data = [] files = [os.path.join(DIRECTORY, file) for file in os.listdir(DIRECTORY)] gt_file = pd.read_csv(GROUND_TRUTH_FILE, sep=',') for file in files: participant_id = file.split("/")[1].split("_")[0] y_train = get_gt_value(gt_file, participant_id) if y_train != -1: # All the participants whose GT is Absent will not be considered. x_train = "" df = pd.read_csv(file, sep='\t') for index, row in df.iterrows(): if row['speaker'] == PARTICIPANT: value = row['value'] x_train = x_train + " " + value print("Data found Participant Id:" + participant_id) data.append([preprocess(x_train), y_train]) data_df = pd.DataFrame(data, columns=[Participant_ID, PHQ8_Score]) data_df.to_csv("Dev-Data", sep=',', index=False)
import numpy as np from importlib import reload import data_preprocessing as dp import prepare_for_model as pfm import other_functions as of import explanatory_analysis as ea import sklearn.linear_model as lr from sklearn.metrics import log_loss, confusion_matrix from sklearn import cross_validation as cv reload(of) reload(pfm) reload(ea) reload(dp) data = pd.read_csv('train.csv') dp.preprocess(data, is_prediction = False, path = 'values_to_keep') data2 = pfm.prepare(data) y = data.OutcomeType timeInc = True model_col = [c for c in data2.columns if not 'AnimalType_' in c \ and (timeInc or (not 'Year_' in c and not 'Month_' in c and not 'Day_' in c and not 'DayOfWeek_' in c and not 'Hour_' in c)) ] ## prepare model for dogs ########################################### X_train, X_test, y_train, y_test = cv.train_test_split( data2.loc[data2.AnimalType_Dog == 1,model_col], y[data2.AnimalType_Dog == 1], test_size=0.0, random_state=13) # test of C parameters
adjM[len(movies) + len(directors) + actor_idx, movie_idx] = 1 if row['actor_2_name'] in actors: actor_idx = actors.index(row['actor_2_name']) adjM[movie_idx, len(movies) + len(directors) + actor_idx] = 1 adjM[len(movies) + len(directors) + actor_idx, movie_idx] = 1 if row['actor_3_name'] in actors: actor_idx = actors.index(row['actor_3_name']) adjM[movie_idx, len(movies) + len(directors) + actor_idx] = 1 adjM[len(movies) + len(directors) + actor_idx, movie_idx] = 1 # print(adjM) # print(adjM.shape) # import scipy.sparse # import os # scipy.sparse.save_npz(os.path.join("data/preprocess/IMDB/", "adjM.npz"), scipy.sparse.csr_matrix(adjM)) # exit() metapath_class_num = 3 metapath_class_list = [[(0, 1, 0), (0, 2, 0)], [(1, 0, 1), (1, 0, 2, 0, 1)], [(2, 0, 2), (2, 0, 1, 0, 2)]] savepath = "data/preprocess/IMDB/" preprocess(adjM=adjM, num_ntypes=metapath_class_num, type_mask=type_mask, expected_metapaths=metapath_class_list, dataoutput=savepath) end = time.time() print(end - begin)
product_quantity['product_month'] = product_quantity['product_date'].apply( lambda x: x[:7]) train_month = product_quantity.groupby(['product_id', 'product_month' ]).sum()['ciiquantity'].unstack() total_null_index = set(product_info.index) - set( train_month.index) #完全缺失值的index #计算相似性矩阵 #data_sim = clt.CalculateSim(product_info) #计算时间较长,可直接将datasim导入Variable explore datasim = pd.read_csv("datasim.csv", index_col='Index') #直接读入,用于填充缺失值 #缺失值填补(均值填补) train_month = prep.fillnan( train_month, 1, 30, 140, datasim) #对大于12个月有空缺值的基于相似度进行填补,其他的填补为140,取topK=30进行填补 #对product_info进行数据预处理 product_info = prep.preprocess(product_info) #计算出每个产品的平均价格 product_quantity['price'].replace(to_replace=-1, method='backfill', inplace=True) product_quantity['price'].replace(to_replace=-1, method='ffill', inplace=True) product_price = product_quantity.groupby(['product_id']).mean()['price'] #对多个ID同时进行训练,采用xgboost result_for_some = pd.DataFrame() result_for_some, feature_importance = trainf.train_for_some( train_month, product_info, train_month.index, product_price, 'xgboost') result_for_some = prep.to_positive(result_for_some.iloc[:, 23:]) #采用均值的方法 average = pd.DataFrame(train_month.iloc[:, 14:23].mean(axis=1),
def __init__(self, K: int = 5): self.K = K self.train_x, self.test_x, self.train_y, self.test_y = preprocess( test_size=0.2)
table_name = 'os_reading_' # 'AUB', 'Reception', 'Meeting', '4thFloor' # AUB first date: 2018-02-02 # Reception first dat: 2018-02-27 # Meeting Room first date: 2018-02-27 # Env Data first date: 2018-02-01 project = 'AUB' osdp = Opensensors('2018-02-27', '2018-05-05', table_name + project, project) data = osdp.data # SQLite database to pandas dataframe conn = sqlite3.connect(table_name + project + ".sqlite") data = preprocess(pd.read_sql_query("SELECT * FROM " + project, conn)) conn.close() #heat = data.iloc[:, 5:] out = Outliers(data) out.plot(5) g = General(data) g.plot_comparison_bars() g.period_plot('month', 'circulation') g.period_plot('month', 'exhibition') g.period_plot('month', 'ai') g.period_plot('month', 'code') g.period_plot('month', 'vr')
import pandas as pd import numpy as np import sklearn.neural_network.multilayer_perceptron import os import data_preprocessing as dp df_dict = dp.rd.read_data_to_dict() train_df = dp.preprocess(df_dict) y = train_df.pop('Score') X = train_df
import data_preprocessing import numpy as np import tensorflow as tf #data processing data1 = data_preprocessing.preprocess('three_samples.txt') data2 = data_preprocessing.preprocess('full_samples.txt') data3 = data_preprocessing.preprocess('null_samples.txt') data = data1 + data2 + data3 data = np.array(data) shuffle_indices = np.random.permutation(np.arange(len(data))) shuffled_data = data[shuffle_indices] train_sample_index = int( 0.8 * float(len(data)) ) training_data,test_data = shuffled_data[:train_sample_index], shuffled_data[train_sample_index:] print('Number of training data samples: {}'.format(len(training_data))) print('Number of test data samples: {}'.format(len(test_data))) test_x, test_y = zip(*test_data) test_x = np.array(test_x) test_y = np.array(test_y) #build graph x = tf.placeholder(tf.float32, shape = [None, 49]) y_ = tf.placeholder(tf.float32, shape = [None, 77]) def weight_variable(shape): initial = tf.truncated_normal(shape,stddev=0.1) return tf.Variable(initial) def bias_variable(shape): initial = tf.constant(0.1, shape=shape) return tf.Variable(initial)
def transform_view(): print("* Requesting data -- API") f = request.files['data_file'] if not f: return ("No file selected. Please choose a CSV file and try again.") stream = io.StringIO(f.stream.read().decode("UTF8"), newline=None) print("stream", stream) csv_input = csv.reader(stream) print("csv input", csv_input) print("* Processing csv_input -- API") df = pd.DataFrame(csv_input, index=None, columns=None) data, df4, df4_column_names, df_normalized, df_normalized_w_target, X_test_new, y_test_new = preprocess( df) print('* Data Preprocessing Complete Flask -- API') print('* Joblib model loaded -- API') # X_train, X_test, y_train, y_test = sample_data(df_normalized_w_target) # print('* Data Sampled') # X_train, X_test_new, y_train, y_test_new = initialize_sample(df_normalized_w_target, X_test, y_test) # print("* Data Initialized for First Pickle") rfc_test_acc, y_pred, class_rept, conf_mat = hyper_param_rf_pickle( X_test_new, y_test_new, model) print("HyperParam Pickle Model", model) print("* Hyperparameter search complete -- API") y_test_new = (y_test_new.as_matrix(columns=None)).astype(int) print("Y Test New Type:", type(y_test_new)) print(y_test_new) print("Y Pred:", type(y_pred)) print(y_pred) conf_mat = confusion_matrix(y_test_new, y_pred) class_rept = classification_report(y_test_new, y_pred) print("* Conf Mat and Class Rept Defined -- API") print("* Saving results in an image....") # return('* CSV File Submitted -- Running API') # full_filename = os.path.join(app.config['UPLOAD_FOLDER'], 'confusion_matrix.png') return render_template("page.html", matrix_image='./Static/confusion_matrix.png')