def run(path):
datasource, types, n_steps, n_channel, n_class, overlap, target, process_num, filter = config_parse(path)
modelname_prefix = '_'.join([datasource, n_steps, n_channel, n_class, overlap, target, filter['phonetype'], filter['phoneposition'], filter['activity']])
n_steps, n_channel, n_class, process_num = map(lambda x : int(x), [n_steps, n_channel, n_class, process_num])
dataconf = DataConf.DataConf(datasource, types, n_steps, n_channel, n_class, float(overlap))
process = DataPreprocess.DataPreprocess(dataconf, process_num, target, phonetype = filter['phonetype'], phoneposition = filter['phoneposition'], activity = filter['activity'])
x_train, y_train, x_valid, y_valid, x_test, y_test = process.load_data(standard=False,issample=False)
# print y_train
if len(y_train) == 0:
return
'''
#for model in ['cnn', 'vgglstm', 'lstm', 'bilstm','vgg']:
#for model in ['cnn', 'vgglstm', 'vgg']:
model = 'cnn'
modelname = "%s#%s" % (model, modelname_prefix)
modelconf = ModelConf.ModelConf(dataconf=dataconf, batch_size=400, learning_rate=0.0001, epochs=50)
modelbuild = ModelBuilder.ModelBuilder(modelconf, modelname, allconfig['target'])
modelbuild.train_lstm(x_train, y_train, x_valid, y_valid, figplot=True)
modelbuild.test(x_test, y_test, ROC=False)
'''
record = []
for model in ['vgglstm']:
p = multiprocessing.Process(target=para_train, args=(model, modelname_prefix, dataconf, target, x_train, y_train, x_valid, y_valid, x_test, y_test))
p.start()
record.append(p)
for process in record:
process.join()
def excutecmd(request):
info = {'msgtype': 'ERR', 'content': []}
if request.method == 'POST':
callback = request.POST.get('callback')
cmd = request.POST.get('cmd')
rid = request.POST.get("rid")
else:
callback = request.GET.get('callback')
cmd = request.GET.get('cmd')
rid = request.GET.get("rid")
ie_key = rid
excute_time = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))
client_ip = request.META['REMOTE_ADDR']
client_ip_locat = IP.find(client_ip)
username = request.user.username
try:
tid = str(random.randint(90000000000000000000, 99999999999999999999))
server = eval(cmd)
cmd = server['cmd']
selectserver = server['selectserver']
if not selectserver: raise IOError("没有选择执行主机")
Data = DataConf.DataConf()
a = threading.Thread(target=cheungssh_web.main,
args=(cmd, ie_key, selectserver, Data, tid))
a.start()
cmd_history = cache.get('cmd_history')
if cmd_history is None: cmd_history = []
allconf = cache.get('allconf')
allconf_t = allconf['content']
for sid in selectserver.split(','):
server_ip = allconf_t[sid]['ip']
cmd_history_t = {
"tid": tid,
"excutetime": excute_time,
"IP": client_ip,
"IPLocat": client_ip_locat,
"user": username,
"servers": server_ip,
"cmd": cmd
}
cmd_history.insert(0, cmd_history_t)
cache.set('cmd_history', cmd_history, 8640000000)
info['msgtype'] = "OK"
except Exception, e:
print "发生错误", e
info['msgtype'] = 'ERR'
info['content'] = str(e)
def run(allconfig):
dataconf = DataConf.DataConf(datasource=allconfig['datasource'], types=allconfig['types'],
n_steps=allconfig['n_steps'], \
n_channels=allconfig['n_channel'], n_class=allconfig['n_class'],
overlap=allconfig['overlap'])
process = DataPreprocess.DataPreprocess(dataconf=dataconf, process_num=allconfig['process_num'],
target=allconfig['target'], \
phonetype=allconfig['condition']['phonetype'],
phoneposition=allconfig['condition']['phoneposition'],
activity=allconfig['condition']['activity'])
x_train, y_train, x_valid, y_valid, x_test, y_test = process.load_data(standard=False)
if len(x_train) == 0:
return
modelname_prefix = '%s_%s_T_%s_P_%s_A_%s_%d_%d_%d_%0.1f' % (allconfig['datasource'], allconfig['target'], \
allconfig['condition']['phonetype'],
allconfig['condition']['phoneposition'],
allconfig['condition']['activity'], \
allconfig['n_class'], allconfig['n_steps'],
allconfig['n_channel'], allconfig['overlap'])
# for model in ['cnn', 'vgglstm', 'lstm', 'bilstm','vgg']:
# for model in ['cnn', 'vgglstm', 'vgg']:
# model = 'vgglstm'
# modelname = "%s#%s" % (model, modelname_prefix)
# # modelname = "cnn#%s" % (modelname_prefix)
# modelconf = ModelConf.ModelConf(dataconf=dataconf, batch_size=400, learning_rate=0.0001, epochs=50)
# modelbuild = ModelBuilder.ModelBuilder(modelconf, modelname, allconfig['target'])
# os.environ['CUDA_VISIBLE_DEVICES']='0'
# modelbuild.train_vgg_lstm(x_train, y_train, x_valid, y_valid, figplot=True)
# modelbuild.test(x_test, y_test, ROC=False)
record = []
for model in ['cnn','vgg','vgglstm','bilstm']:
p = multiprocessing.Process(target=para_train, args=(model, modelname_prefix, dataconf, x_train, y_train, x_valid, y_valid, x_test, y_test))
p.start()
record.append(p)
for process in record:
process.join()
def collect(id='all'):
cmd = {
"platform": "/usr/sbin/dmidecode -s system-product-name|tail -1",
"os": """awk 'NR==1{print}' /etc/issue""",
"bit": "getconf LONG_BIT",
"cpu_num": "grep 'physical id' /proc/cpuinfo| sort| uniq| wc -l",
"core_num": """grep "processor" /proc/cpuinfo| wc -l""",
"cpu_type": """grep name /proc/cpuinfo| cut -f2 -d: | uniq -c""",
"serial_num":
"""/usr/sbin/dmidecode -s system-serial-number|sed 's/ //g'""",
"interface": """lspci|grep 'Ethernet'""",
"mem": """/usr/bin/free -m|awk '$1=="Mem:"{printf("%sMB",$2)}'""",
"disk": """parted -l | grep -E '^Disk' | grep -v mapper""",
"mac": """/sbin/ifconfig -a |awk '$0~/HWaddr/{print $NF}'""",
"hostname": "hostname",
"swap": """/usr/bin/free -m|awk '$1=="Swap:"{printf("%sMB",$2)}'"""
}
for a in cmd.keys():
Data = DataConf.DataConf()
cheungssh_web.main(cmd[a], 'all-ie', id, Data, 'hardware', a)
def excutecmd(request):
info={'msgtype':'ERR','content':[]}
callback=request.GET.get('callback')
cmd=request.GET.get('cmd')
rid=request.GET.get("rid")
ie_key=rid
excute_time=time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(time.time()))
client_ip=request.META['REMOTE_ADDR']
client_ip_locat=IP.find(client_ip)
username=request.user.username
try:
server=eval(cmd)
cmd=server['cmd']
selectserver=server['selectserver']
Data=DataConf.DataConf()
a=threading.Thread(target=cheungssh_web.main,args=(cmd,ie_key,selectserver,Data))
a.start()
info['msgtype']="OK"
except Exception,e:
info['content']=str(e)
plt.ylabel('acc-loss')
plt.legend(loc="upper right")
plt.savefig("%s/acc-loss.svg" % save_path)
plt.show()
if __name__ == '__main__':
model = "har_before_transfer"
path = "./model.conf"
datasource, types, n_steps, n_channel, n_class, overlap, target, process_num, filter = config_parse(path)
modelname_prefix = '_'.join(
[datasource, n_steps, n_channel, n_class, overlap, target, filter['phonetype'], filter['phoneposition'],
filter['activity']])
n_steps, n_channel, n_class, process_num = map(lambda x: int(x), [n_steps, n_channel, n_class, process_num])
dataconf = DataConf.DataConf(datasource, types, n_steps, n_channel, n_class, float(overlap))
process = DataPreprocess.DataPreprocess(dataconf, process_num, target, phonetype=filter['phonetype'],
phoneposition=filter['phoneposition'], activity=filter['activity'])
x_train, y_train, x_valid, y_valid, x_test, y_test = process.load_data(standard=False)
modelname = "%s#%s" % (model, modelname_prefix)
modelconf = ModelConf.ModelConf(dataconf=dataconf, batch_size=300, learning_rate=0.0001, epochs=70)
modelbuild = Keras_ModelBuilder(modelconf, modelname, target)
modelbuild.train_vgg_lstm(x_train, y_train, x_valid, y_valid, figplot=True)
modelbuild.test(x_test, y_test, 'transfer_before_vgg_lstm', ROC=False)
model = "hasc_after_transfer"
if __name__ == '__main__':
model = "authen-cnn-svm"
path = "./model.conf"
datasource, types, n_steps, n_channel, n_class, overlap, target, process_num, filter = config_parse(
path)
modelname_prefix = '_'.join([
datasource, n_steps, n_channel, n_class, overlap, target,
filter['phonetype'], filter['phoneposition'], filter['activity']
])
n_steps, n_channel, n_class, process_num = map(
lambda x: int(x), [n_steps, n_channel, n_class, process_num])
multi_dataconf = DataConf.DataConf(datasource, types, n_steps, n_channel,
n_class, float(overlap))
multi_process = DataPreprocess.DataPreprocess(
multi_dataconf,
process_num,
target,
phonetype=filter['phonetype'],
phoneposition=filter['phoneposition'],
activity=filter['activity'])
x_train1, y_train1, x_valid1, y_valid1, x_test1, y_test1 = multi_process.load_data(
standard=False)
multi_modelname = "%s#%s" % (model, modelname_prefix)
multi_modelconf = ModelConf.ModelConf(dataconf=multi_dataconf,
batch_size=500,
learning_rate=0.0001,
epochs=300)