def mulThreadpredict(test_data, model, start_index_lab=270):
preP = PreProcess()
test_X, test_loc_dic = preP.preProcessTestData(test_data)
test_Xc = preP.getFeatureScaler(test_X)
test_X = None
length = len(test_Xc)
# 初始化线程池
print "cpu number:", cpu_count()
cpus = cpu_count()
pool = threadpool.ThreadPool(cpus - 1)
print "test data len:", length
print "predict result ... ... "
list_args = []
for i in xrange(start_index_lab, length):
pre_result = None
start_index = i * 1000
end_index = (i + 1) * 1000
if end_index > length:
end_index = length
args = [
model, test_data, test_Xc, start_index, end_index, test_loc_dic
]
list_args.append((args, None))
requests = threadpool.makeRequests(threadPredict, list_args)
[pool.putRequest(req) for req in requests]
pool.wait()
def main():
print "Start......"
rf = ReadCsvFile()
# train_data = rf.ReadTrainFile()
preP = PreProcess()
# train_X,train_lab,loc_dic = preP.preProcessTrainData(train_data)
# train_Xc = preP.getFeatureScaler(train_X)
# print "Train model"
# TrainModel(train_Xc,train_lab)
# train_lab = None
# loc_dic = None
# train_data = None
# train_X = None
# train_Xc = None
# with open(Config.ResultDataPath+"result.csv","w") as fp:
# print "清空文件","result.csv"
model = LoadModel()
# print "Read data predict ... ... "
test_data = rf.ReadTestFile()
test_X,test_loc_dic = preP.preProcessTestData(test_data)
test_Xc = preP.getFeatureScaler(test_X)
test_X = None
length = len(test_Xc)
print "test data len:",length
print "predict result ... ... "
for i in xrange(104,length):
pre_result = None
start_index = i * 100
end_index = (i+1) * 100
if end_index > length:
end_index = length
start_time = time.time()
pre_result = predict(model,test_Xc[start_index:end_index])
end_time = time.time()
print "predict from %d to %d. \n cost time :%lf s\n" % (start_index, end_index,end_time-start_time)
start_time = time.time()
result = transformResult(pre_result, test_loc_dic)
logging.info("predict from %d to %d. \ncost time :%lf s" % (start_index, end_index,end_time-start_time))
wr = WriteResult()
wr.WriteResultAnswer(test_data[start_index:end_index], result,Config.ResultDataPath+"result_1.csv")
end_time = time.time()
logging.info("transform index from %d to %d. \ncost time :%lf s" % (start_index, end_index,end_time-start_time))
print "transform index from %d to %d. \n cost time :%lf s\n" % (start_index, end_index,end_time-start_time)
def mulThreadpredict(test_data,model,start_index_lab=270):
preP = PreProcess()
test_X, test_loc_dic = preP.preProcessTestData(test_data)
test_Xc = preP.getFeatureScaler(test_X)
test_X = None
length = len(test_Xc)
# 初始化线程池
print "cpu number:", cpu_count()
cpus = cpu_count()
pool = threadpool.ThreadPool(cpus-1)
print "test data len:", length
print "predict result ... ... "
list_args = []
for i in xrange(start_index_lab, length):
pre_result = None
start_index = i * 1000
end_index = (i + 1) * 1000
if end_index > length:
end_index = length
args = [model,test_data,test_Xc,start_index,end_index,test_loc_dic]
list_args.append((args,None))
requests = threadpool.makeRequests(threadPredict, list_args)
[pool.putRequest(req) for req in requests]
pool.wait()