def main():
    pipeline = Pipeline([
        ('vect', TfidfVectorizer()),
        ('clf', LogisticRegression())
    ])
    parameters = {
        'vect__max_df': (0.25, 0.5, 0.75),
        'vect__stop_words': ('english', None),
        'vect__max_features': (5000, 10000, None),
        'vect__ngram_range': ((1, 1), (1, 2)),
        'vect__use_idf': (True, False),
        'vect__norm': ('l1', 'l2'),
        'clf__penalty': ('l1', 'l2'),
        'clf__C': (0.1, 1, 10),
    }
    df = pd.read_csv('movie-reviews/train.tsv', header=0, delimiter='\t')
    X, y = df['Phrase'], df['Sentiment'].as_matrix()
    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.5)
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='accuracy')
    grid_search.fit(X_train, y_train)
    print 'Best score: %0.3f' % grid_search.best_score_
    print 'Best parameters set:'
    best_parameters = grid_search.best_estimator_.get_params()
    for param_name in sorted(parameters.keys()):
        print '\t%s: %r' % (param_name, best_parameters[param_name])

    predictions = grid_search.predict(X_test)
    print 'Accuracy:', accuracy_score(y_test, predictions)
    print 'Precision:', precision_score(y_test, predictions)
    print 'Recall:', recall_score(y_test, predictions)
    print 'F1 score:', f1_score(y_test, predictions)
示例#2
0
文件: mvrnn.py 项目: 5idaidai/MVRNN
    def evaluate(self, params, rnnDataTest):
#        predictLabels = np.zeros(len(rnnDataTest.allSNum), dtype='int32')
#        probabilities = np.zeros(len(rnnDataTest.allSNum))
        predictLabels = []
        trueLabels = []
        
        allSNum = rnnDataTest.allSNum
        allSTree = rnnDataTest.allSTree
        allSStr = rnnDataTest.allSStr
        verbIndices = rnnDataTest.verbIndices
#        allSNN = rnnDataTest.allSNN
#        allIndicies = rnnDataTest.allIndicies
        sentenceLabels = rnnDataTest.sentenceLabels
    
        ndoc = rnnDataTest.ndoc()
        print "Total number of trees/sentences to be evaluated: ", ndoc
        for s in range(ndoc):              
            if(s % 100 == 0) :
                print "Processing sentences ", s , ' - ', s+100   
            thissentVerbIndices = verbIndices[s]  
            sStr = allSStr[s]; sNum = allSNum[s]; sTree = allSTree[s]
            labels = sentenceLabels[s]
            if((len(sNum) == 1) or (len(thissentVerbIndices)==0) or (labels.shape[1] != len(sStr))):
                continue  #only one word in a sent, no verbs for this sent, tokens and labels mismatch                 
            for nverb, vid in enumerate(thissentVerbIndices):
                for wid in range(len(sStr)):
                    indices = np.array([vid, wid])   
                    truelabel = labels[nverb, wid]  
                    setVerbnWordDistanceFeat(self.Wv, sNum, vid, wid, params) 
                    tree = forwardPropTree(self.W, self.WO, self.Wcat, self.Wv, self.Wo, sNum, sTree, sStr, sNN=None, indicies=None,  params=params) 
                    trueLabels.append(truelabel)       
                    calPredictions(tree, self.Wcat, self.Wv, indices, sStr, params) #updates score, nodepath etc for this verb, word pair
                    predictedLabel = np.argmax(tree.y)
                    predictLabels.append(predictedLabel)
        
        f1 = f1_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#, labels=all_labels)
        p = precision_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#, labels=all_labels)
        r = recall_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#), labels=all_labels)
        print "XXXXXXX F1 = ", f1
        print "XXXXXXX P = ", p
        print "XXXXXXX R = ", r
        print 
        return predictLabels
示例#3
0
 def evaluate(self, params, rnnDataTest):
     predictLabels = []
     trueLabels = []        
     
     allSNum = rnnDataTest.allSNum
     allSTree = rnnDataTest.allSTree
     allSStr = rnnDataTest.allSStr
     verbIndices = rnnDataTest.verbIndices
     sentenceLabels = rnnDataTest.sentenceLabels
 
     ndoc = rnnDataTest.ndoc()
     print "Total number of trees/sentences to be evaluated: ", ndoc
     for s in range(ndoc):              
         if(s % 100 == 0) :
             print "Processing sentences ", s , ' - ', s+100   
         thissentVerbIndices = verbIndices[s]  
         sStr = allSStr[s]; sNum = allSNum[s]; sTree = allSTree[s]
         labels = sentenceLabels[s]
         if((len(sNum) == 1) or (len(thissentVerbIndices)==0) or (labels.shape[1] != len(sStr))):
             continue  #only one word in a sent, no verbs for this sent, tokens and labels mismatch                 
         for nverb, vid in enumerate(thissentVerbIndices):
             scoresMat = np.zeros((len(sStr), self.Wcat.shape[0]))
             for wid in range(len(sStr)):
                 indices = np.array([vid, wid])   
                 setVerbnWordDistanceFeat(self.Wv, sNum, vid, wid, params) 
                 tree = forwardPropTree(self.W, self.WO, self.Wcat, self.Wv, self.Wo, sNum, sTree, sStr, sNN=None, indicies=None,  params=params) 
                 calPredictions(tree, self.Wcat, self.Wv, indices, sStr, params) #updates score, nodepath etc for this verb, word pair
                 scoresMat[wid,:] = tree.score
             pred_answer = viterbi(scoresMat, self.Tran)
             true_answer = labels[nverb,:]
             for i in range(len(pred_answer)):
                 predictLabels.append(pred_answer[i])
                 trueLabels.append(true_answer[i])
             #TODO : calculate predicted labels     
     
     f1 = f1_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#, labels=all_labels)
     p = precision_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#, labels=all_labels)
     r = recall_score(y_true=trueLabels, y_pred=predictLabels, pos_label=None)#), labels=all_labels)
     print "XXXXXXX F1 = ", f1
     print "XXXXXXX P = ", p
     print "XXXXXXX R = ", r
     print 
     return predictLabels
示例#4
0
	for tweet in reader[0:2*(numironicos/3)]:
		tweets_train.append(tweet["text"])
		labels_train.append("noironia")
	for tweet in reader[2*(numironicos/3):]:
		tweets_test.append(tweet["text"])
		labels_test.append("noironia")

stop_words = []
f = open("spanish.txt") 
for line in f:
	stop_words.append(line.strip())

f.close()

y_train = np.array(labels_train, dtype=object) 
y_test = np.array(labels_test, dtype=object) 

vectorizer = TfidfVectorizer(input='content', max_df=0.5, stop_words = stop_words)
X_train = vectorizer.fit_transform(np.array(tweets_train, dtype=object))
X_test = vectorizer.transform(np.array(tweets_test, dtype=object))
classifier = RandomForestClassifier(n_estimators = 10)
classifier.fit(X_train.toarray(), y_train)
prediction = classifier.predict(X_test.toarray())

print '\nAccuracy :', accuracy_score(y_test, prediction)
print '\nPrecision :', precision_score(y_test, prediction)
print '\nRecall :', recall_score(y_test, prediction)
print '\nF-score :', f1_score(y_test, prediction)
print '\nClasification report:\n', classification_report(y_test,prediction)
print '\nConfussion matrix :\n',confusion_matrix(y_test, prediction)
示例#5
0
from feature_extraction import extract_features

## Vars
force_train = True
enable_plot = False

## Load and split the data in train and test sets
print "Loading data..."
X, y = load_numbers()
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)

## Get trained classifier
print "Training classifier..."
clf = load_or_train(X_train, y_train, force_train, enable_plot)
print clf

## Compute the features of the test set and predict
print "Predicting test set..."
features = extract_features(X_test)
y_pred = clf.predict(features)

print y_test
print y_pred

## Score
f1 = f1_score(y_test, y_pred)
print "f1-score for is {}%".format(f1)
#if enable_plot:
print_classification_report(y_test, y_pred)
plot_confusion_matrix(y_test, y_pred, range(0, 10))
示例#6
0
def fit_predict(config, X_train, y_train, X_test=None, y_test=None, ref_thd=None):
    """
    Uses the configuration dictionary settings to train a model using the
    specified training algorithm. If set, also evaluates the trained model 
    in a test set. Additionally, performs feature selection and model parameters
    optimization.
    
    @param config: the configuration dictionary obtained parsing the 
    configuration file.
    @param X_train: the np.array object for the matrix containing the feature
    values for each instance in the training set.
    @param y_train: the np.array object for the response values of each instance
    in the training set.
    @param X_test: the np.array object for the matrix containing the feature
    values for each instance in the test set. Default is None.
    @param y_test: the np.array object for the response values of each instance
    in the test set. Default is None.
    """
    # sets the selection method
    transformer = set_selection_method(config)

    # if the system is configured to run feature selection
    # runs it and modifies the datasets to the new dimensions
    if transformer is not None:
        log.info("Running feature selection %s" % str(transformer))

        log.debug("X_train dimensions before fit_transform(): %s,%s" % X_train.shape)
        log.debug("y_train dimensions before fit_transform(): %s" % y_train.shape)

        X_train = transformer.fit_transform(X_train, y_train)

        log.debug("Dimensions after fit_transform(): %s,%s" % X_train.shape)

        if X_test is not None:
            X_test = transformer.transform(X_test)

    # sets learning algorithm and runs it over the training data
    estimator, scorers = set_learning_method(config, X_train, y_train)
    log.info("Running learning algorithm %s" % str(estimator))
    estimator.fit(X_train, y_train)

    if (X_test is not None) and (y_test is not None):
        log.info("Predicting unseen data using the trained model...")
        y_hat = estimator.predict(X_test)
        log.info("Evaluating prediction on the test set...")
        for scorer_name, scorer_func in scorers:
            v = scorer_func(y_test, y_hat)
            log.info("%s = %s" % (scorer_name, v))
        log.info("Customized scores: ")
        try:
            log.info("pearson_corrcoef = %s" % pearson_corrcoef(y_test, y_hat))
        except:
            pass
        try:
            log.info("Precision score: = %s" % precision_score(y_test, y_hat))
        except:
            pass
        try:
            log.info("Recall score: = %s" % recall_score(y_test, y_hat))
        except:
            pass
        try:
            log.info("F1 score: = %s" % f1_score(y_test, y_hat))
        except:
            pass
        try:
            log.info("MAE: = %s" % mean_absolute_error(y_test, y_hat))
        except:
            pass
        try:
            log.info("RMSE: = %s" % root_mean_squared_error(y_test, y_hat))
        except:
            pass
        try:
            res = classify_report_bin(y_test, y_hat)
            if "N/A" <> res:
                log.info("Classify report bin: = %s" % res)
            else:
                res = classify_report_bin_regression(y_test, y_hat)
                if "N/A" <> res:
                    log.info("Classify report bin regression: = %s" % res)
                else:
                    if ref_thd is None:
                        log.error("No ref thd defined")
                    else:
                        refthd = float(ref_thd)
                        res = classify_report_regression(y_test, y_hat, refthd)
                        log.info("Classify report regression: = %s" % res)
        except Exception, e:
            print e
        with open("predicted.csv", "w") as _fout:
            for _x, _y in zip(y_test, y_hat):
                print >> _fout, "%f\t%f" % (_x, _y)
示例#7
0
    le = preprocessing.LabelEncoder()
    le.fit(ids_)
    verbose("Total classes",le.classes_.shape[0])
    ids=le.transform(ids_)

    X_train, X_test, y_train, y_test=\
        train_test_split(feats, ids, test_size=0.20, random_state=42) 
    
    verbose("Training")
    classifier=RandomForestClassifier(
            n_estimators=opts.estimators,
            n_jobs=opts.nprocessors,
            max_depth=20,
            verbose=True)

    # Aprendiendo
    classifier.fit(X_train, y_train)

    # Prediciendo
    verbose("Prediction")
    prediction = classifier.predict(X_test)

    print( 'Accuracy              :', accuracy_score(y_test, prediction))
    print( 'Precision             :', precision_score(y_test, prediction))
    print( 'Recall                :', recall_score(y_test, prediction))
    print( 'F-score               :', f1_score(y_test, prediction))
    print( '\nClasification report:\n', classification_report(y_test,
            prediction))
    print( '\nConfussion matrix   :\n',confusion_matrix(y_test, prediction))
            y_.extend(y_test)
            prediction_.extend(prediction)



    verbose('----------\n')
    verbose("Evaluation")

    if opts.mode in ['age','gender']:
        from sklearn.metrics.metrics import precision_score, recall_score, confusion_matrix, classification_report, accuracy_score, f1_score
        # Calculando desempeño
        print( 'Accuracy              :', accuracy_score(y_, prediction_))
        print( 'Precision             :', precision_score(y_, prediction_))
        print( 'Recall                :', recall_score(y_, prediction_))
        print( 'F-score               :', f1_score(y_, prediction_))
        print( '\nClasification report:\n', classification_report(y_,
                prediction_))
        print( '\nConfussion matrix   :\n',confusion_matrix(y_, prediction_))
    else:
        from sklearn.metrics.metrics import mean_absolute_error, mean_squared_error,r2_score
        print( 'Mean Abs Error        :', mean_absolute_error(y_, prediction_))
        print( 'Mean Sqr Error        :', mean_squared_error(y_, prediction_))
        print( 'R2 Error              :', r2_score(y_, prediction_))


    #plots:
    #import matplotlib.pyplot as plt
    #confusion_matrix_plot = confusion_matrix(y_test, prediction)
    #plt.title('matriz de confusion')
    #plt.colorbar()