def tfidf_cloud(n_trees): dio = DataIO("/data/Settings_cloud.json") submission = False min_samples_split = 2 param = """Normal count vector with max 200. New submission which is repeatable. and nicer count_vector_titles = TfidfVectorizer( read_column(train_filename, column_name), max_features=200, norm='l1', smooth_idf=True, sublinear_tf=False, use_idf=True) """ if submission: type_n = "train_full" type_v = "valid_full" else: type_n = "train" type_v = "valid" #features = dio.join_features("%s_" + type_n + "_tfidf_matrix_max_f_200", #["Title", "FullDescription", "LocationRaw"], #extra_features) #validation_features = dio.join_features("%s_" + type_v + "_tfidf_matrix_max_f_200", #["Title", "FullDescription", "LocationRaw"], #extra_valid_features) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_features", features) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_valid_features", validation_features) def load(filename): return joblib.load(path_join("/data", filename)) features = load("train_200f_noNorm_categoryTimeType_tfidfl1_features_jl") validation_features = load( "train_200f_noNorm_categoryTimeType_tfidfl1_valid_features_jl") print "features", features.shape print "valid features", validation_features.shape #salaries = dio.get_salaries(type_n, log=True) #if not submission: #valid_salaries = dio.get_salaries(type_v, log=True) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_salaries", salaries) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries", valid_salaries) #joblib.dump(salaries, "train_200f_noNorm_categoryTimeType_tfidfl2_salaries_jl", compress=5) #joblib.dump(valid_salaries, "train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries_jl", compress=5) #TODO: valid salaries so narobe dumpane salaries = load("train_200f_noNorm_categoryTimeType_tfidfl2_salaries_jl") valid_salaries = load( "train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries_jl") dio.is_log = True print salaries.shape name = "ExtraTree_min_sample%d_%dtrees_200f_noNorm_categoryTimeType_tfidfl1_new_log" % ( min_samples_split, n_trees) print name #dio.save_prediction("testni", np.array([1,2,3]), type_n="testno") classifier = ExtraTreesRegressor( n_estimators=n_trees, verbose=2, n_jobs=4, # 2 jobs on submission / 4 on valid test oob_score=False, min_samples_split=min_samples_split, random_state=3465343) #dio.save_model(classifier, "testni_model", 99.) classifier.fit(features, salaries) predictions = classifier.predict(validation_features) if submission: dio.save_prediction(name, predictions, type_n=type_v) dio.write_submission(name + ".csv", predictions=predictions) else: dio.compare_valid_pred(valid_salaries, predictions) metric = dio.error_metric mae = metric(valid_salaries, predictions) print "MAE validation: ", mae dio.save_model(classifier, name, mae) dio.save_prediction(name, predictions, type_n=type_v)
#if bin_n > 4 : #make_grid_search(MultinomialNB(), {"alpha": [0.01, 0.1, 0.5, 1]}, "multinomialnb" + nm, "Multinomial NB" + par) #make_grid_search(SGDClassifier(), {'n_iter': [50, 100, 150], 'penalty': ['l2', 'l1']}, "sgd_class" + nm, "SGDClassifier classes" + par) ##make_grid_search(KNeighborsClassifier(), {'n_neighbors': range(4,100,20)}, "kneighbour" + nm, "Kneighbour classes" + par) #make_grid_search(RandomForestClassifier(random_state=42), {'min_samples_split': [2, 30]}, "randomForest" + nm, "Random Forest" + par) ##make_grid_search(GradientBoostingClassifier(), {'learning_rate': [0.1, 0.5], 'subsample': [1,0.8,0.6], 'n_estimators':[100,150]}, "GBM" + nm, "Gradient Boosting Machines " + par) bin_n = 4 salaries_enc = encode_salaries(salaries, bin_n) valid_salaries_enc = encode_salaries(valid_salaries, bin_n) nm = "_tfidf_titleFullLoc_bin%d" % bin_n model_name = "randomForest" + nm par = " classed from 0-11500 then %d classes to 100 000 and to end\n Tfidf of Title full and location Raw" % (bin_n) classifier = RandomForestClassifier(min_samples_split=2, random_state=42) classifier.fit(features, salaries_enc) prediction = classifier.predict(validation_features) dio.save_prediction(model_name, prediction, "valid_classes") dio.save_model(classifier, model_name, parameters=par) print (classification_report(valid_salaries_enc, prediction)) print confusion_matrix(valid_salaries_enc, prediction) prediction = classifier.predict(features) dio.save_prediction(model_name, prediction, "train_classes") print (classification_report(salaries_enc, prediction)) print confusion_matrix(salaries_enc, prediction)
classifier = ExtraTreesRegressor( n_estimators=n_trees, #classifier = RandomForestRegressor(n_estimators=n_trees, verbose=2, n_jobs=4, # 2 jobs on submission / 4 on valid test oob_score=True, min_samples_split=min_samples_split, random_state=3465343) classifier.fit(features, salaries) #classifier = dio.load_model(name) #predictions = classifier.predict(validation_features) metric = dio.error_metric mae = metric(valid_salaries, predictions) print "MAE validation: ", mae dio.save_model(classifier, name, mae, parameters=param) dio.save_prediction(name, predictions, type_n=type_v) importances = classifier.feature_importances_ std = np.std([tree.feature_importances_ for tree in classifier.estimators_], axis=0) indices = np.argsort(importances)[::-1] f_names = map(lambda x: "Title(%d)" % x, range(1, 201)) f_names.extend(map(lambda x: "Desc(%d)" % x, range(1, 201))) f_names.extend(map(lambda x: "LocR(%d)" % x, range(1, 201))) f_names.extend(columns) num_feat = len(f_names) # Print the feature ranking print "Feature ranking:" for f in xrange(len(indices)):
min_samples_split=min_samples_split, random_state=3465343) print features[salaries_idx[0], :].shape print salaries[salaries_idx].shape classifier.fit(features[salaries_idx[0], :], salaries[salaries_idx]) predictions_part = classifier.predict(validation_features[valid_idx[0]]) return predictions_part, valid_idx predictions = np.zeros_like(valid_salaries) for cur_class_id in range(num_classes + 1): predictions_part, idx = predict(cur_class_id) if idx is not None: predictions[idx] = predictions_part print "Part MAE: ", metric(valid_salaries[idx], predictions_part) if submission: dio.save_prediction(name, predictions, type_n=type_v) dio.write_submission(name + ".csv", predictions=predictions) else: dio.compare_valid_pred(valid_salaries, predictions) metric = dio.error_metric mae = metric(valid_salaries, predictions) print "MAE validation: ", mae dio.save_model(ExtraTreesRegressor(), name, mae) dio.save_prediction(name, predictions, type_n=type_v) #oob_predictions = classifier.oob_prediction_ #mae_oob = mean_absolute_error(salaries, oob_predictions) #print "MAE OOB: ", mae_oob #classifier1 = ExtraTreesRegressor(n_estimators=n_trees, #verbose=1, #n_jobs=3, #oob_score=False,
"vowpall_loc5" ] #model_names = [model2, model4] #model_names = [model1, model6, model4] #fit_predict(model2) #fit_predict(model1) #fit_predict(model3) #fit_predict(model5) #fit_predict(model4, features, salaries, validation_features, type_n="test_subm") model_name = "predictions_submit_test.txt" predictions = np.loadtxt(path_join(dio.data_dir, "code", "from_fastml", "optional", model_name)) dio.save_prediction("vowpall_submission", predictions, type_v) model_name = "predictions_submit_test_loc5.txt" predictions = np.loadtxt(path_join(dio.data_dir, "code", "from_fastml", "optional", model_name)) dio.save_prediction("vowpall_loc5", predictions, type_v) all_model_predictions = [] for model_name in model_names: #fit_predict(model_name, features, salaries, validation_features, type_n="test_subm") model_predictions = dio.get_prediction(model_name=model_name, type_n="test_full") if not model_name.endswith("log") and not model_name.startswith("vowpall"): model_predictions = np.log(model_predictions) print "modelp", model_predictions.shape #print "%s\nMAE: %f\n" % (model_name, log_mean_absolute_error(np.log(valid_salaries), model_predictions)) all_model_predictions.append(model_predictions) predictions = np.vstack(all_model_predictions).T
#make_grid_search(MultinomialNB(), {"alpha": [0.01, 0.1, 0.5, 1]}, "multinomialnb" + nm, "Multinomial NB" + par) #make_grid_search(SGDClassifier(), {'n_iter': [50, 100, 150], 'penalty': ['l2', 'l1']}, "sgd_class" + nm, "SGDClassifier classes" + par) ##make_grid_search(KNeighborsClassifier(), {'n_neighbors': range(4,100,20)}, "kneighbour" + nm, "Kneighbour classes" + par) #make_grid_search(RandomForestClassifier(random_state=42), {'min_samples_split': [2, 30]}, "randomForest" + nm, "Random Forest" + par) ##make_grid_search(GradientBoostingClassifier(), {'learning_rate': [0.1, 0.5], 'subsample': [1,0.8,0.6], 'n_estimators':[100,150]}, "GBM" + nm, "Gradient Boosting Machines " + par) bin_n = 4 salaries_enc = encode_salaries(salaries, bin_n) valid_salaries_enc = encode_salaries(valid_salaries, bin_n) nm = "_tfidf_titleFullLoc_bin%d" % bin_n model_name = "randomForest" + nm par = " classed from 0-11500 then %d classes to 100 000 and to end\n Tfidf of Title full and location Raw" % ( bin_n) classifier = RandomForestClassifier(min_samples_split=2, random_state=42) classifier.fit(features, salaries_enc) prediction = classifier.predict(validation_features) dio.save_prediction(model_name, prediction, "valid_classes") dio.save_model(classifier, model_name, parameters=par) print(classification_report(valid_salaries_enc, prediction)) print confusion_matrix(valid_salaries_enc, prediction) prediction = classifier.predict(features) dio.save_prediction(model_name, prediction, "train_classes") print(classification_report(salaries_enc, prediction)) print confusion_matrix(salaries_enc, prediction)
def tfidf_cloud(n_trees): dio = DataIO("/data/Settings_cloud.json") submission = False min_samples_split = 2 param = """Normal count vector with max 200. New submission which is repeatable. and nicer count_vector_titles = TfidfVectorizer( read_column(train_filename, column_name), max_features=200, norm='l1', smooth_idf=True, sublinear_tf=False, use_idf=True) """ if submission: type_n = "train_full" type_v = "valid_full" else: type_n = "train" type_v = "valid" #features = dio.join_features("%s_" + type_n + "_tfidf_matrix_max_f_200", #["Title", "FullDescription", "LocationRaw"], #extra_features) #validation_features = dio.join_features("%s_" + type_v + "_tfidf_matrix_max_f_200", #["Title", "FullDescription", "LocationRaw"], #extra_valid_features) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_features", features) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_valid_features", validation_features) def load(filename): return joblib.load(path_join("/data", filename)) features = load("train_200f_noNorm_categoryTimeType_tfidfl1_features_jl") validation_features = load("train_200f_noNorm_categoryTimeType_tfidfl1_valid_features_jl") print "features", features.shape print "valid features", validation_features.shape #salaries = dio.get_salaries(type_n, log=True) #if not submission: #valid_salaries = dio.get_salaries(type_v, log=True) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_salaries", salaries) #np.save("train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries", valid_salaries) #joblib.dump(salaries, "train_200f_noNorm_categoryTimeType_tfidfl2_salaries_jl", compress=5) #joblib.dump(valid_salaries, "train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries_jl", compress=5) #TODO: valid salaries so narobe dumpane salaries = load("train_200f_noNorm_categoryTimeType_tfidfl2_salaries_jl") valid_salaries = load("train_200f_noNorm_categoryTimeType_tfidfl2_valid_salaries_jl") dio.is_log = True print salaries.shape name = "ExtraTree_min_sample%d_%dtrees_200f_noNorm_categoryTimeType_tfidfl1_new_log" % (min_samples_split, n_trees) print name #dio.save_prediction("testni", np.array([1,2,3]), type_n="testno") classifier = ExtraTreesRegressor(n_estimators=n_trees, verbose=2, n_jobs=4, # 2 jobs on submission / 4 on valid test oob_score=False, min_samples_split=min_samples_split, random_state=3465343) #dio.save_model(classifier, "testni_model", 99.) classifier.fit(features, salaries) predictions = classifier.predict(validation_features) if submission: dio.save_prediction(name, predictions, type_n=type_v) dio.write_submission(name + ".csv", predictions=predictions) else: dio.compare_valid_pred(valid_salaries, predictions) metric = dio.error_metric mae = metric(valid_salaries, predictions) print "MAE validation: ", mae dio.save_model(classifier, name, mae) dio.save_prediction(name, predictions, type_n=type_v)