def __train(self):
print 'training neural network ...'
funcUtil.recordStatus(self.__id, 'training neural network ...')
train_s_time = time.time()
try:
self.__bpNN.train()
except Exception, ex:
print ex
funcUtil.write_log('train')
self.__success = False
def __preProcess(self):
print 'pre processing data ...'
funcUtil.recordStatus(self.__id, 'pre processing data ...')
process_s_time = time.time()
try:
self.__bpNN.preProcess()
except Exception, ex:
print ex
funcUtil.write_log('preProcess')
self.__success = False
def __load(self):
print 'loading data ...'
funcUtil.recordStatus(self.__id, 'loading data ...')
load_s_time = time.time()
try:
o_load = load.Load(self.__dataConfig) # 加载数据 并 进行数据预处理
data_training, data_val, data_test, data_last = o_load.run()
self.__bpNN.loadData(data_training, data_val, data_test, data_last)
except Exception, ex:
print ex
funcUtil.write_log('load')
self.__success = False
def __recordResult(self):
if not self.__record:
return
print 'recording result ...'
funcUtil.recordStatus(self.__id, 'recording result ...')
record_s_time = time.time()
try:
result = {
'training_record': self.__bpNN.trainingRecord,
'test_record': self.__bpNN.testRecord,
'original_data': self.__bpNN.originalData,
'pred': self.__bpNN.pred,
}
result = json.dumps(result)
with open(funcUtil.getTmpPath(self.__id, 'result.tmp'), 'w') as f:
f.write(result)
except Exception, ex:
print ex
funcUtil.write_log('recordResult')
# -*- coding: utf-8 -*-
import sys, os
sys.path.append(os.path.split(os.path.abspath(os.path.curdir))[0])
import master
import config.network as configNetwork
import config.load as configLoad
import lib.funcUtil as funcUtil
# unique_id = sys.argv[1]
unique_id = 'test'
funcUtil.killRunningProcess(unique_id)
funcUtil.recordPid(unique_id)
config = {
'data': configLoad.config(), # 读取 加载数据的配置
'network': configNetwork.config(), # 读取 神经网络的配置
'id': unique_id,
'record': True,
}
o_master = master.Master(config)
o_master.run()
funcUtil.recordStatus(unique_id, 'done')
funcUtil.recordPid(unique_id, 'done')
import time
print 'done ' + str(time.time())
load_s_time = time.time()
try:
o_load = load.Load(self.__dataConfig) # 加载数据 并 进行数据预处理
data_training, data_val, data_test, data_last = o_load.run()
self.__bpNN.loadData(data_training, data_val, data_test, data_last)
except Exception, ex:
print ex
funcUtil.write_log('load')
self.__success = False
load_e_time = time.time()
print 'finish loading'
print 'load data use time: %s\n' % str(load_e_time - load_s_time)
funcUtil.recordStatus(
self.__id,
'finish loading (use time: %s)' % str(load_e_time - load_s_time))
def __preProcess(self):
print 'pre processing data ...'
funcUtil.recordStatus(self.__id, 'pre processing data ...')
process_s_time = time.time()
try:
self.__bpNN.preProcess()
except Exception, ex:
print ex
funcUtil.write_log('preProcess')
self.__success = False
process_e_time = time.time()
print 'process %d finish' % process_id
return [master_config, run_result]
if __name__ == '__main__':
process_num = 30
import sys
# unique_id = sys.argv[1]
unique_id = 'test'
s_time = time.time()
funcUtil.killRunningProcess(unique_id)
funcUtil.recordPid(unique_id)
funcUtil.recordStatus(unique_id, 'Start ...')
p = Pool()
result_list = []
for i in range(process_num):
result_list.append(p.apply_async(run, args=(i, process_num, unique_id)))
print 'Waiting for all subprocesses done...'
p.close()
p.join()
print 'All subprocesses done.'
result_list = map(lambda x: x.get(), result_list)
def compare_accuracy(a, b):