/
Keyword.py
86 lines (65 loc) · 2.51 KB
/
Keyword.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""
https://www.topcoder.com/challenge-details/30054204/?type=develop&nocache=true
"""
import sys
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import RidgeClassifierCV
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.externals import joblib
from sklearn import cross_validation
from sklearn.cross_validation import KFold
import pandas as pd
import numpy as np
def predict(test_data, out_file):
df=pd.read_excel(test_data)
df = df.fillna('')
x1=list(df["#water_tech"].str.lower())
x2=list(df["#water_source"].str.lower())
X=list(zip(*[x1,x2]))
clf = joblib.load('classifier.pkl')
y = clf.predict(X)
df["Water Source Types"] = y
df.to_excel(out_file, index=False)
def train(training_data):
df=pd.read_excel(training_data)
df = df.fillna('')
x1=list(df["#water_tech"].str.lower())
x2=list(df["#water_source"].str.lower())
X=list(zip(*[x1,x2]))
y=list(df["Categorized"])
clf = GClassifier().fit(X, y)
joblib.dump(clf, 'classifier.pkl')
def test(training_data):
df=pd.read_excel(training_data)
df = df.fillna('')
x1=list(df["#water_tech"].str.lower())
x2=list(df["#water_source"].str.lower())
X=list(zip(*[x1,x2]))
y=list(df["Categorized"])
kf = KFold(len(df), n_folds=4)
X=np.array(X)
y=np.array(y)
print(cross_validation.cross_val_score(GClassifier(), X, y, cv=kf))
class GClassifier(BaseEstimator, ClassifierMixin):
def __init__(self):
pass
def fit(self, X, y):
trainx1, trainx2 = zip(*X)
self.count_vect = CountVectorizer(analyzer='word', ngram_range=(1, 2))
self.count_vect.fit(list(trainx1)+list(trainx2))
X_train_counts1 = self.count_vect.transform(trainx1)
X_train_counts2 = self.count_vect.transform(trainx2)
X_train_counts = np.concatenate((X_train_counts1.toarray(),X_train_counts2.toarray()),axis=1)
self.clf = RidgeClassifierCV().fit(X_train_counts, y)
return self
def predict(self, X):
testx1, testx2 = zip(*X)
X_test_counts1 = self.count_vect.transform(testx1)
X_test_counts2 = self.count_vect.transform(testx2)
X_test_counts = np.concatenate((X_test_counts1.toarray(),X_test_counts2.toarray()),axis=1)
return self.clf.predict(X_test_counts)
if __name__ == "__main__":
if len(sys.argv)==2:
train(sys.argv[1])
elif len(sys.argv)==3:
predict(sys.argv[1], sys.argv[2])