def predict(self, test):
test = categories.createCategorical(test)
results = self.model.predict(test)
final = []
for result in results:
final.append(np.argmax(result))
return final
def trainModel(self, train, y):
train = categories.createCategorical(train)
#estimator = KerasClassifier(build_fn=self.model, epochs=10, batch_size=1000, verbose=1)
self.model.fit(train,
np.eye(4)[y.ravel().tolist()],
epochs=100,
batch_size=1000,
verbose=1)
def predict(self, test):
test = categories.createCategorical(test)
test1 = self.normalize(test)
return self.model.predict(test1).tolist()
def trainModel(self, train, y):
train = categories.createCategorical(train)
self.fitNormalizer(train)
train = self.normalize(train)
self.model.fit(train, y)
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.tree import RandomForest
from pyspark.mllib.evaluation import MulticlassMetrics
conf = SparkConf()
sc = SparkContext(conf=conf)
train_df = pd.read_csv('../data/train_new.csv',usecols = ['STAT_CAUSE_DESCR','LATITUDE','LONGITUDE','DISCOVERY_DATE','FIRE_SIZE','avg_temp'])
y = pd.DataFrame()
y['STAT_CAUSE_DESCR']=train_df['STAT_CAUSE_DESCR']
y=labels.createLabel(y)
y['STAT_CAUSE_DESCR']=y['STAT_CAUSE_DESCR'].astype(int)
train_df = train_df.drop(columns=['STAT_CAUSE_DESCR'])
train_df=categories.createCategorical(train_df)
pd.concat([y,train_df],axis=1,sort=False).reset_index(drop=True).to_csv('../data/traintemp.csv',header=False,index=False)
train = sc.textFile("../data/traintemp.csv").map(lambda line: line.split(","))
def parsePoint(line):
return LabeledPoint(line[0], line[1:])
parsedData = train.map(lambda line:parsePoint(line))
model =RandomForest.trainClassifier(parsedData,seed=42,numClasses=4,numTrees=200,maxDepth=10,\
categoricalFeaturesInfo={5:2,6:2,7:2,8:2,9:2,10:2,11:2,12:2,13:2,14:2,15:2,16:2,17:2,18:2,19:2,20:2,21:2})