def generate_trace(self, min_length=5):
# We start the sentence with the start token
x = np.zeros((1, self.seq_length, self.vocab_size))
mask = np.zeros((1, self.seq_length))
new_sentence = []
i = 0
# Repeat until we get an end token
selIndex = self.word_to_index[IN_SELECTION_TOKEN]
notSelIndex = self.word_to_index[NOT_IN_SELECTION_TOKEN]
while not ((len(new_sentence) > 0) and
((new_sentence[-1] == selIndex) or
(new_sentence[-1] == notSelIndex))):
probs = self.propabilities(x, mask)[0]
# samples = np.random.multinomial(1, probs)
# index = np.argmax(samples)
index = np.random.choice(range(len(probs)), p=probs)
new_sentence.append(index)
x[0, i, index] = 1
mask[0, i] = 1
i += 1
# Seomtimes we get stuck if the sentence becomes too long, e.g. "........" :(
# And: We don't want sentences with UNKNOWN_TOKEN's
if len(
new_sentence
) >= self.seq_length or index == self.word_to_index[UNKNOWN_TOKEN]:
writeLog("Generated exceedingly long example trace. Skipping.")
return None
if len(new_sentence) < min_length:
return None
res = [self.index_to_word[x] for x in new_sentence]
writeLog("Generated example trace of length %d: %s" %
(len(res), str(res)))
return res
def main(opts):
if opts.openworld == False:
datalist = ReadWebList()
datalen = len(datalist)
else:
datalist = ReadOpenWebList(
5000, 1) # 5000 sites for open world dataset, each with 1 instance
datalen = len(datalist)
print("len datalist for openworld = ", len(datalist))
for i in range(0, datalen, cm.MAX_SITES_PER_TOR_PROCESS):
if i + cm.MAX_SITES_PER_TOR_PROCESS < datalen:
writeLog("data start from %s to %s" %
(datalist[i][0],
datalist[i + cm.MAX_SITES_PER_TOR_PROCESS - 1][0]))
print("data start from %s to %s\n" %
(datalist[i][0],
datalist[i + cm.MAX_SITES_PER_TOR_PROCESS - 1][0]))
launch_tor_with_custom_stem(
datalist[i:i + cm.MAX_SITES_PER_TOR_PROCESS], opts.browser)
else:
writeLog("data start from %s to %s" %
(datalist[i][0], datalist[-1][0]))
print("data start from %s to %s\n" %
(datalist[i][0], datalist[-1][0]))
launch_tor_with_custom_stem(datalist[i:], opts.browser)
ParsePcapFile()
def trainModel(self, callback):
data_size = len(self.positions_train)
writeLog("Training...")
p = 0
num_iterations = 0
num_iterations_after_report = 0
num_report_iterations = 1
avg_cost = 0
# writeLog("It: " + str(data_size * self.num_epochs // self.batch_size))
try:
it = 0
while (num_report_iterations <= self.num_callbacks):
x, y, mask = self.gen_data(self.TS_train, p,
self.positions_train,
self.batch_size)
it += 1
p += self.batch_size
num_iterations += self.batch_size
num_iterations_after_report += self.batch_size
# if(p+self.batch_size+self.seq_length >= data_size):
# writeLog('Carriage Return')
# p = 0;
avg_cost += self.train(x, y, mask)
if (callback and num_iterations_after_report >=
self.num_iterations_between_reports):
callback(num_iterations, it, avg_cost / it,
num_report_iterations)
avg_cost = 0
num_iterations_after_report = num_iterations_after_report - self.num_iterations_between_reports
num_report_iterations = num_report_iterations + 1
# callback(num_iterations, it, avg_cost / it, num_report_iterations)
except KeyboardInterrupt:
pass
def __init__(self, case_name, dataset_name, algorithm, num_layers,
optimizer, learning_rate, batch_size, num_callbacks,
hidden_dim_size, num_iterations_between_reports,
grad_clipping, predict_only_outcome, final_trace_only,
trace_length_modifier, max_num_words, truncate_unknowns, rng):
writeLog("Using data set: " + dataset_name)
self.algorithm = algorithm
self.num_layers = num_layers
self.optimizer = optimizer
self.learning_rate = learning_rate
self.batch_size = batch_size
self.num_callbacks = num_callbacks
self.traces = trace_registry[dataset_name](trace_length_modifier)
self.dataset_size = len(self.traces)
self.dataset_name = dataset_name
self.case_name = case_name
self.hidden_dim_size = hidden_dim_size
self.num_iterations_between_reports = num_iterations_between_reports
self.grad_clipping = grad_clipping
self.rng = rng
self.predict_only_outcome = predict_only_outcome
self.final_trace_only = final_trace_only
self.trace_length_modifier = trace_length_modifier
self.max_num_words = max_num_words
self.truncate_unknowns = truncate_unknowns
lasagne.random.set_rng(rng)
try:
self.createModel()
except:
writeLog("Exception: " + sys.exc_info()[0])
def TBBSetup(driverpath, controller, idx):
driver = 0
try:
driver = TorBrowserDriver(driverpath, tor_cfg=cm.USE_STEM)
except Exception as e:
writeLog("[crawl.py error]TBBSetup error: " + str(e))
print("[crawl.py error]TBBSetup error")
print(str(e))
driver = 0
return driver
def reportTemperature(self, morning = True):
rtn = self.session.post(self.reader.get('temperature', 'getstateUrl'))
rtn = json.loads(rtn.content)
if len(rtn['module']) > 0 and rtn['module'][0]['sfdt'] == "1":
#self.pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 添加体温失败",
# "今日添加体温失败，添加时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
# "上报用户名: " + self.usr + "\n"
# + "错误原因：" + '已经生成今日体温')
tokenstr = rtn['module'][0]["id"]
else:
token_url = self.reader.get('temperature', 'token-url')
tokenstr = bytes.decode(self.session.post(token_url).content)
addUrl = self.reader.get('temperature', 'addUrl')
result = self.session.post(addUrl, data = {"info": json.dumps({'token': tokenstr}, ensure_ascii=False)})
tokenstr = json.loads(result.content)['module']
if not json.loads(result.content)['isSuccess']:
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 添加体温失败",
"今日添加体温失败，添加时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr + "\n" + "错误原因：" + json.loads(result.content)['msg'], self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
#StateById = self.reader.get('temperature', 'getstateByid')
#rtn = json.loads(self.session.post(StateById, data = {"info": json.dumps({'id': tokenstr}, ensure_ascii=False)}).content)['module']
updateUrl = addUrl = self.reader.get('temperature', 'update-url')
data = {
"id": tokenstr,
"sdid1": "1",
"tw1" : "37.2",
"fr1" : "0",
"bs" : "1"
}
rtn1 = json.loads(self.session.post(updateUrl, data={"info": json.dumps({"data": data}, ensure_ascii=False)}).content)['isSuccess']
data = {
"id": tokenstr,
"sdid2": "3",
"tw2" : "37.2",
"fr2" : "0",
"bs" : "2"
}
rtn2 = json.loads(self.session.post(updateUrl, data={"info": json.dumps({"data": data}, ensure_ascii=False)}).content)['isSuccess']
if rtn1 and rtn2:
writeLog('success', time.strftime("%m-%d", time.localtime()), "tempLog.txt")
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 体温上报成功",
"今日体温上报成功，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr, self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
else:
writeLog('fail', time.strftime("%m-%d", time.localtime()), "tempLog.txt")
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 体温上报失败",
"今日体温上报失败，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr, self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
def cleanupStream(controller, crawlcnt, domain):
print("check & remove existing streams...")
l = []
for stream in controller.get_streams():
l.append(stream.circ_id)
d = getGuard(controller, l)
for stream in controller.get_streams():
try:
# print("stream id: ",stream.id,stream.circ_id,stream.target_address)
writeStreamInfo("%s,%s,%s,%s,%s,%s,%s,%s" %
(domain, crawlcnt, stream.id, stream.circ_id,
d[stream.circ_id], stream.target_address,
stream.target, str(stream.target_port)))
controller.close_stream(stream.id)
except Exception as e:
writeLog("### error in closing stream: " + str(stream.id))
pass
def TorSetup(tor_binary):
tor_process, controller = 0, 0
print("in tor setup binary = ", tor_binary)
try:
tor_process = launch_tbb_tor_with_stem(tbb_path=cm.driverpath,
torrc=cm.TorConfig,
tor_binary=tor_binary)
controller = Controller.from_port(
port=int(cm.TorConfig['ControlPort']))
controller.authenticate()
print("getting tor circuit...")
print("write entry guard/ circuit to log...")
except Exception as e:
print("[crawl.py error]TorSetup: " + str(e) + "\n")
writeLog("[crawl.py error]TorSetup: " + str(e) + "\n")
tor_process, controller = 0, 0
return tor_process, controller
def ParsePcapFile(filepath, NodeList, outputpath):
fw = open(outputpath, 'w')
fw.write(','.join(cm.packetinfo) + '\n') # write header
writeLog("parse filepath = %s" % (filepath))
fp = open(filepath, 'rb')
try:
pcap = dpkt.pcap.Reader(fp)
cnt = 0
for ts, buf in pcap:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.ip
ipsrc = inet_to_str(ip.src)
ipdst = inet_to_str(ip.dst)
if ipsrc in NodeList or ipdst in NodeList:
output = retrieveInfo(cnt, ts, ip)
fw.write(','.join([str(x) for x in output]) + "\n")
# fw.write('%s,%s,%s,%s,%s\n'%(str(ts),str(ipaddr),str(dstaddr),str(flag),str(datalen)))
cnt += 1
except Exception as e:
writeLog("[pcaputils.py]ParsePcapFile error: %s" % (str(e)))
finally:
fp.close()
def report(self):
rtn = self.session.post(self.addUrl)
# print(rtn.url)
try:
status = rtn.json()
except:
status = {"msg": "网页可能发生了错误"}
writeLog('fail', time.strftime("%m-%d", time.localtime()))
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 上报失败",
"今日上报失败，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr + "\n"
+ "错误原因：" + status['msg'], self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
return False
print(status)
rtn = self.session.post(
self.editUrl, data={'info': json.dumps({'id': status['module']})})
data = rtn.json()['module']['data'][0]
if 'brzgtw' in data:
data['brzgtw'] = '36.3'
rtn = self.session.post(
self.saveUrl, data={'info': json.dumps({'model': data})})
try:
print(rtn.text)
status = rtn.json()
if(status['isSuccess']):
writeLog('success', time.strftime("%m-%d", time.localtime()))
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 上报成功",
"今日上报成功，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr, self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
return True
else:
writeLog('fail', time.strftime("%m-%d", time.localtime()))
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 上报失败",
"今日上报失败，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr + "\n"
+ "错误原因：辅导员已经审查", self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
return False
except:
writeLog('fail', time.strftime("%m-%d", time.localtime()))
pushMsg(time.strftime("%m-%d %H:%M:%S", time.localtime()) + " 上报失败",
"今日上报失败，上报时间为" + time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + "\n" +
"上报用户名: " + self.usr + "\n"
+ "错误原因：" + status['msg'], self.pushUrl, self.reader.get('Push', 'token'), self.reader.get('Push', 'template'))
return False
def train(modelConfig,dataConfig,logConfig):
"""
训练
:param modelConfig: 模型配置
:param dataConfig: 数据配置
:param logConfig: 日志配置
:return:
"""
# 模型配置
model = modelConfig['model']
criterion = modelConfig['criterion']
optimzer = modelConfig['optimzer']
epochs = modelConfig['epochs']
device = modelConfig['device']
#数据加载器
trainLoader = dataConfig['trainLoader']
validLoader = dataConfig['validLoader']
trainLength = dataConfig['trainLength']
validLength = dataConfig['validLength']
# 日志及模型保存
modelPath = logConfig['modelPath']
historyPath = logConfig['historyPath']
logPath = logConfig['logPath']
lastModelPath = logConfig['lastModelPath']
trainLosses = []
trainAcces = []
validLosses = []
validAcces = []
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print('train is starting in ' + now)
bestAcc = 0.
for epoch in range(epochs):
print("Epoch{}/{}".format(epoch, epochs))
print("-" * 10)
trainLoss, trainAcc = oneEpoch_train(model,trainLoader,optimzer,criterion,device)
validLoss, validAcc = oneEpoch_valid(model,validLoader,criterion,device)
trainLoss = trainLoss / len(trainLoader)
trainAcc = trainAcc / trainLength
validLoss = validLoss / len(validLoader)
validAcc = validAcc / validLength
# trainLosses.append(trainLoss)
# trainAcces.append(trainAcc)
#
# validLosses.append(validLoss)
# validAcces.append(validAcc)
# 模型验证有进步时,保存模型
if validAcc > bestAcc:
bestAcc = validAcc
# saveModel(model,modelPath)
# 训练日志
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
trainLog = now + " Train loss is :{:.4f},Train accuracy is:{:.4f}%\n".format(trainLoss, 100 * trainAcc)
validLog = now + " Valid loss is :{:.4f},Valid accuracy is:{:.4f}%\n".format(validLoss, 100 * validAcc)
bestlog = now + ' bestAcc is {:.4f}%'.format(100 * bestAcc)
log = trainLog + validLog
print(log+bestlog)
# 训练历史 每个EPOCH都覆盖一次
# history = {
# 'trainLosses':trainLosses,
# 'trainAcces':trainAcces,
# 'validLosses':validLosses,
# 'validAcces':validAcces
# }
writeLog(logPath,log)
def report(num_examples_seen, it, avg_cost, num_report_iterations):
t2 = time.time()
tutrain = (t2 - self.previous_time)
self.cumul_train_time = self.cumul_train_time + tutrain
self.time_used.append(tutrain)
self.generate_trace(5)
self.sr_examplesSeen.append(num_examples_seen)
self.cms = {}
self.cms_str = ""
writeLog("Testing 100% training samples")
sr_train = calculateSuccessRate(self.traces_train, 1.0, 0)
self.sr_trains.append(sr_train)
writeLog("Testing 100% test samples")
sr_test = calculateSuccessRate(self.traces_test, 1.0, 1)
writeLog("Testing 75% test samples")
sr_tests_75p = calculateSuccessRate(self.traces_test, 0.75, 2)
writeLog("Testing 50% test samples")
sr_tests_50p = calculateSuccessRate(self.traces_test, 0.5, 3)
writeLog("Testing 25% test samples")
sr_tests_25p = calculateSuccessRate(self.traces_test, 0.25, 4)
self.sr_tests.append(sr_test)
self.sr_tests_75p.append(sr_tests_75p)
self.sr_tests_50p.append(sr_tests_50p)
self.sr_tests_25p.append(sr_tests_25p)
self.avg_costs.append(avg_cost)
data_size = len(self.TS_train)
epoch = it * self.batch_size / data_size
t3 = time.time()
tutest = (t3 - t2)
self.cumul_test_time = self.cumul_test_time + tutest
self.previous_time = t3
self.time_used_for_test.append(tutest)
self.all_cms.append(self.cms)
writeLog(
"Iteration: %i (%i) Total time used: ~%f seconds (train: %f, test: %f)"
% (num_report_iterations, num_examples_seen,
(time.time() - self.start_time) * 1., self.cumul_train_time,
self.cumul_test_time))
writeLog("Epoch {} average loss = {}".format(epoch, avg_cost))
writeLog(
"Success rates: test: %f test 75%%: %f test 50%%: %f test 25%%: %f train: %f"
%
(sr_test, sr_tests_75p, sr_tests_50p, sr_tests_25p, sr_train))
writeResultRow([
datetime.now().replace(microsecond=0).isoformat(), "ok", "",
self.case_name, self.dataset_name, self.dataset_size,
self.algorithm, self.num_layers, self.hidden_dim_size,
self.optimizer, self.learning_rate, self.seq_length,
self.batch_size, self.grad_clipping,
self.num_iterations_between_reports, num_report_iterations,
num_examples_seen, epoch, tutrain, self.cumul_train_time,
tutest, self.cumul_test_time, sr_train, sr_test, sr_tests_75p,
sr_tests_50p, sr_tests_25p, avg_cost, self.auc, self.cms[1][0],
self.cms[1][1], self.cms[1][2], self.cms[1][3],
str(self.cms_str), self.predict_only_outcome,
self.final_trace_only, self.trace_length_modifier,
self.num_iterations_between_reports *
self.num_callbacks == 100000 * 50, self.max_num_words,
self.truncate_unknowns
])
def launch_tor_with_custom_stem(datalist, browser):
print("length of data: ", len(datalist))
tor_binary = join(cm.TorProxypath, cm.DEFAULT_TOR_BINARY_PATH)
tor_process, controller = 0, 0
try:
TRYTOR_CNT = cm.TRYCNT
while TRYTOR_CNT > 0 and tor_process == 0 and controller == 0:
print("try to setup tor:", str(TRYTOR_CNT))
tor_process, controller = TorSetup(tor_binary)
TRYTOR_CNT -= 1
if tor_process == 0:
raise TorSetupError
print("finish tor proxy setup...")
xvfb_display = start_xvfb() # virtual display
for ele in datalist:
t = getTime()
savepath, out_img = SetOutputPath(ele, t)
p = 0
try:
driver, TRYCNT = 0, cm.TRYCNT
while driver == 0 and TRYCNT != 0:
print("try to setup tbb:", str(TRYCNT))
args = (cm.driverpath, controller,
ele[2]) if browser == 'TBB' else ()
options = {
'TBB': TBBSetup,
'FF': FFSetup,
'CR': ChromeSetup
}
driver = options[browser](*args)
TRYCNT -= 1
if driver == 0:
raise TBBSetupError
cmd = "tcpdump -i %s tcp and not port ssh -w %s" % (
cm.netInterface, savepath)
print('cmd = ', cmd)
cmd = cmd.split(' ')
p = subprocess.Popen(cmd)
try:
timeout(cm.VISITPAGE_TIMEOUT)
driver.get('https://' + ele[0])
cancel_timeout()
time.sleep(cm.DURATION_VISIT_PAGE)
p.terminate()
if (ele[2] == 0 or ele[2] == 2):
driver.get_screenshot_as_file(out_img)
writeLog(str(t) + "," + ele[0] + "," + str(ele[2]))
print("Finish tcpdump sleep...")
except TimeExceededError:
writeLog("Error crawling," + ele[0] + "," + str(ele[2]) +
"\n" + str("Page visit Timeout"))
finally:
cancel_timeout()
except TBBSetupError:
print("[crawl.py error]: unable to setup TBB")
writeLog("[crawl.py error]: unable to setup TBB")
except Exception as e:
with open(cm.ErrorFilePath, 'a+') as fw:
fw.write(ele[0] + "," + str(e) + "\n")
writeLog("Error crawling," + ele[0] + "," + str(ele[2]) +
"\n" + str(e))
finally:
if p != 0 and p.returncode != 0:
try:
p.terminate()
except Exception as e:
writeLog("[crawl.py] tcpdump terminate error: " +
str(e))
if controller != 0:
cleanupStream(controller, str(ele[2]), ele[0])
if driver != 0:
try:
timeout(30)
driver.quit()
cancel_timeout()
except Exception as e:
cancel_timeout()
writeLog("[crawl.py] driver quit error: " + str(e))
if ele[2] != 3:
time.sleep(cm.PAUSE_BETWEEN_INSTANCES)
else:
time.sleep(cm.PAUSE_BETWEEN_SITES)
RemoveTmpFile()
RemoveProcess()
except TorSetupError:
print("[crawl.py] unable to set up tor proxy")
writeLog("[crawl.py] unable to set up tor proxy")
except Exception as e:
print("[crawl.py]launch_tor_with_custom_stem Error")
print("Error:", str(e))
writeLog("[crawl.py]launch_tor_with_custom_stem Error : " + str(e))
finally:
if tor_process != 0:
tor_process.kill()
stop_xvfb(xvfb_display)
def run(self):
"""Run the main loop."""
while (True):
self._captureManager.enterFrame()
frame = self._captureManager.frame
if frame is not None:
if (self.autoBackgroundImg == None):
self.autoBackgroundImg = numpy.float32(frame)
rawFrame = frame.copy()
# First work out the region of interest by
# subtracting the fixed background image
# to create a mask.
#print frame
#print self._background_depth_img
absDiff = cv2.absdiff(frame, self._background_depth_img)
benMask, maskArea = filters.getBenMask(absDiff, 8)
cv2.accumulateWeighted(frame, self.autoBackgroundImg, 0.05)
# Convert the background image into the same format
# as the main frame.
#bg = self.autoBackgroundImg
bg = cv2.convertScaleAbs(self.autoBackgroundImg, alpha=1.0)
# Subtract the background from the frame image
cv2.absdiff(frame, bg, frame)
# Scale the difference image to make it more sensitive
# to changes.
cv2.convertScaleAbs(frame, frame, alpha=100)
# Apply the mask so we only see the test subject.
frame = cv2.multiply(frame, benMask, dst=frame, dtype=-1)
if (maskArea <= self.cfg.getConfigInt('area_threshold')):
bri = (0, 0, 0)
else:
# Calculate the brightness of the test subject.
bri = filters.getMean(frame, benMask)
# Add the brightness to the time series ready for analysis.
self._ts.addSamp(bri[0])
self._ts.addImg(rawFrame)
# Write timeseries to a file every 'output_framecount' frames.
if (self._outputFrameCount >=
self.cfg.getConfigInt('output_framecount')):
# Write timeseries to file
self._ts.writeToFile("%s/%s" % \
( self.cfg.getConfigStr('output_directory'),
self.cfg.getConfigStr('ts_fname')
))
self._outputFrameCount = 0
else:
self._outputFrameCount = self._outputFrameCount + 1
# Only do the analysis every 15 frames (0.5 sec), or whatever
# is specified in configuration file analysis_framecount
# parameter.
if (self._frameCount <
self.cfg.getConfigInt('analysis_framecount')):
self._frameCount = self._frameCount + 1
else:
# Look for peaks in the brightness (=movement).
self._nPeaks, self._ts_time, self._rate = self._ts.findPeaks(
)
#print "%d peaks in %3.2f sec = %3.1f bpm" % \
# (nPeaks,ts_time,rate)
oldStatus = self._status
if (maskArea > self.cfg.getConfigInt('area_threshold')):
# Check for alarm levels
if (self._rate > self.cfg.getConfigInt("rate_warn")):
self._status = self.ALARM_STATUS_OK
elif (self._rate >
self.cfg.getConfigInt("rate_alarm")):
self._status = self.ALARM_STATUS_WARN
else:
self._status = self.ALARM_STATUS_FULL
else:
self._status = self.ALARM_STATUS_NOT_FOUND
if (oldStatus == self.ALARM_STATUS_OK and
self._status == self.ALARM_STATUS_WARN) or \
(oldStatus == self.ALARM_STATUS_WARN and
self._status == self.ALARM_STATUS_FULL):
# Write timeseries to file
self._ts.writeToFile("%s/%s" % \
( self.cfg.getConfigStr('output_directory'),
self.cfg.getConfigStr('alarm_ts_fname')
),bgImg=self._background_depth_img)
# Collect the analysis results together and send them
# to the web server.
resultsDict = {}
resultsDict['fps'] = "%3.0f" % self.fps
resultsDict['bri'] = "%4.0f" % self._ts.mean
resultsDict['area'] = "%6.0f" % maskArea
resultsDict['nPeaks'] = "%d" % self._nPeaks
resultsDict['ts_time'] = self._ts_time
resultsDict['rate'] = "%d" % self._rate
resultsDict['time_t'] = time.ctime()
resultsDict['status'] = self._status
self._ws.setAnalysisResults(resultsDict)
# Write the results to file as a json string
utils.writeJSON(resultsDict,"%s/%s" % \
(self._tmpdir,
self.cfg.getConfigStr("data_fname")))
utils.writeLog(resultsDict,"%s/%s" % \
(self._tmpdir,
"benFinder_alarms.log"))
# Plot the graph of brightness, and save the images
# to disk.
self._ts.plotRawData(
file=True,
fname="%s/%s" % \
(self._tmpdir,self.cfg.getConfigStr("chart_fname")))
cv2.imwrite(
"%s/%s" % (self._tmpdir,
self.cfg.getConfigStr("raw_image_fname")),
rawFrame)
cv2.imwrite(
"%s/%s" %
(self._tmpdir,
self.cfg.getConfigStr("masked_image_fname")), frame)
self._frameCount = 0
else:
print "Null frame received - assuming end of file and exiting"
break
self._captureManager.exitFrame()
def initializeTraces(self):
word_to_index = []
index_to_word = []
TRAIN_SIZE = int(self.dataset_size * TRAIN_SAMPLE_PERCENTAGE)
TEST_SIZE = int(self.dataset_size * (1 - TRAIN_SAMPLE_PERCENTAGE))
indexes = self.rng.permutation(self.dataset_size)
# indexes = range(self.dataset_size)
self.traces_train = self.traces[indexes[:TRAIN_SIZE]]
self.traces_test = self.traces[indexes[TRAIN_SIZE:]]
# Tokenize the sentences into words
writeLog("Tokenizing %s sentences." % len(self.traces))
# tokenized_sentences = [nltk.word_tokenize(trace.sentence) for trace in traces]
tokenized_sentences_train = [
nltk.WhitespaceTokenizer().tokenize(trace.sentence)
for trace in self.traces_train
]
tokenized_sentences = [
nltk.WhitespaceTokenizer().tokenize(trace.sentence)
for trace in self.traces
]
# Count the word frequencies
word_freq = nltk.FreqDist(itertools.chain(*tokenized_sentences_train))
writeLog("Found %d unique words tokens." % len(word_freq.items()))
# Get the most common words and build index_to_word and word_to_index vectors
vocab = sorted(word_freq.items(),
key=lambda x: (x[1], x[0]),
reverse=True)
writeLog("Using vocabulary size %d." % len(vocab))
writeLog(
"The least frequent word in our vocabulary is '%s' and appeared %d times."
% (vocab[-1][0], vocab[-1][1]))
words = []
for x in vocab:
w = x[0]
if (w != IN_SELECTION_TOKEN and w != NOT_IN_SELECTION_TOKEN):
words.append(w)
words = np.asarray(words)
if ((self.max_num_words != None)
and (self.max_num_words < len(words))):
words = words[range(self.max_num_words)]
writeLog(
"Vocabulary was truncated to %d most frequent words in training set."
% len(words))
index_to_word = np.concatenate(
[[UNKNOWN_TOKEN, IN_SELECTION_TOKEN, NOT_IN_SELECTION_TOKEN],
words])
word_to_index = dict([(w, i) for i, w in enumerate(index_to_word)])
self.seq_length = 0
# Replace all words not in our vocabulary with the unknown token
for i, sent in enumerate(tokenized_sentences):
ts = [w if w in word_to_index else UNKNOWN_TOKEN for w in sent]
if (self.truncate_unknowns):
origts = ts
ts = []
wasUnknown = False
for w in origts:
isUnknown = w == UNKNOWN_TOKEN
if ((not isUnknown) or (not wasUnknown)):
ts.append(w)
wasUnknown = isUnknown
tokenized_sentences[i] = ts
l = len(tokenized_sentences[i])
if (l > self.seq_length):
self.seq_length = l
writeLog("Maximum sequence length is %d tokens." % (self.seq_length))
self.word_to_index = word_to_index
self.index_to_word = index_to_word
self.vocab_size = len(self.word_to_index)
tokenized_sentences = np.asarray(tokenized_sentences)
self.TS_train = tokenized_sentences[indexes[:TRAIN_SIZE]]
self.positions_train = []
if (self.final_trace_only):
for i, ts in enumerate(self.TS_train):
l = len(ts)
if l > 1:
self.positions_train.append([i, l - 1])
else:
for i, ts in enumerate(self.TS_train):
l = len(ts)
if l > 1:
for pos in range(l - 1):
self.positions_train.append([i, pos])
self.TS_test = tokenized_sentences[indexes[TRAIN_SIZE:]]
self.positions_test = []
for i, ts in enumerate(self.TS_test):
l = len(ts)
if l > 1:
for pos in range(l - 1):
self.positions_test.append([i, pos])
loss = nn.partial_fit(xs)
# log = '%s batch: %10d cost: %.8e nmi-1: %.8f nmi-2: %.8f nmi-3: %.8f' % (
# dt.datetime.now(), i, loss, 0, 0, 0)
# print(log)
if not epoch % 1:
loss, dys = nn.predict(X)
nmi1 = 0.5
nmi2 = 0.5
nmi3 = 0.5
log = '%s epoch: %10d cost: %.8e nmi-1: %.8f nmi-2: %.8f nmi-3: %.8f' % (
dt.datetime.now(), epoch, loss, nmi1, nmi2, nmi3)
utils.writeLog('../log', name, log)
print(log)
# dhs = np.histogram(dys, bins=outdim)[0]
# print(khs)
# print(dhs)
if not epoch % 100:
nn.saveModel('../model', epoch)
epoch += 1
flag = False
if flag: break
print("optimization is finished! ")
def createModel(self):
self.initializeTraces()
writeLog("Preparing " + str(self.num_layers) +
" layers for algorithm: " + self.algorithm)
# First, we build the network, starting with an input layer
# Recurrent layers expect input of shape
# (batch size, SEQ_LENGTH, num_features)
mask_var = T.matrix('mask')
l_in = lasagne.layers.InputLayer(shape=(None, None, self.vocab_size))
l_mask = lasagne.layers.InputLayer((None, None), mask_var)
l_layers = [l_in]
# We now build the LSTM layer which takes l_in as the input layer
# We clip the gradients at GRAD_CLIP to prevent the problem of exploding gradients.
if (self.algorithm == "gru"):
layerCreatorFunc = lambda parentLayer, isFirstLayer, isLastLayer: lasagne.layers.GRULayer(
parentLayer,
self.hidden_dim_size,
grad_clipping=self.grad_clipping,
mask_input=l_mask if isFirstLayer else None,
only_return_final=isLastLayer)
else:
# All gates have initializers for the input-to-gate and hidden state-to-gate
# weight matrices, the cell-to-gate weight vector, the bias vector, and the nonlinearity.
# The convention is that gates use the standard sigmoid nonlinearity,
# which is the default for the Gate class.
# gate_parameters = lasagne.layers.recurrent.Gate(
# W_in=lasagne.init.Orthogonal(), W_hid=lasagne.init.Orthogonal(),
# b=lasagne.init.Constant(0.))
# cell_parameters = lasagne.layers.recurrent.Gate(
# W_in=lasagne.init.Orthogonal(), W_hid=lasagne.init.Orthogonal(),
# # Setting W_cell to None denotes that no cell connection will be used.
# W_cell=None, b=lasagne.init.Constant(0.),
# # By convention, the cell nonlinearity is tanh in an LSTM.
# nonlinearity=lasagne.nonlinearities.tanh)
layerCreatorFunc = lambda parentLayer, isFirstLayer, isLastLayer: lasagne.layers.LSTMLayer(
parentLayer,
self.hidden_dim_size,
grad_clipping=self.grad_clipping,
mask_input=l_mask if isFirstLayer else None,
nonlinearity=lasagne.nonlinearities.tanh,
# Here, we supply the gate parameters for each gate
# ingate=gate_parameters, forgetgate=gate_parameters,
# cell=cell_parameters, outgate=gate_parameters,
# We'll learn the initialization and use gradient clipping
only_return_final=isLastLayer)
for layerId in range(self.num_layers):
l_layers.append(
layerCreatorFunc(l_layers[layerId], layerId == 0,
layerId == self.num_layers - 1))
# The output of l_forward_2 of shape (batch_size, N_HIDDEN) is then passed through the softmax nonlinearity to
# create probability distribution of the prediction
# The output of this stage is (batch_size, vocab_size)
l_out = lasagne.layers.DenseLayer(
l_layers[len(l_layers) - 1],
num_units=self.vocab_size,
W=lasagne.init.Normal(),
nonlinearity=lasagne.nonlinearities.softmax)
l_layers.append(l_out)
# Theano tensor for the targets
target_values = T.ivector('target_output')
#! target_var = T.matrix('target_output')
# lasagne.layers.get_output produces a variable for the output of the net
network_output = lasagne.layers.get_output(l_out)
# https://github.com/Lasagne/Lasagne/blob/master/examples/recurrent.py
# The network output will have shape (n_batch, 1); let's flatten to get a
# 1-dimensional vector of predicted values
# predicted_values = network_output.flatten()
# flat_target_values = target_values.flatten()
# Our cost will be mean-squared error
# cost = T.mean((predicted_values - flat_target_values)**2)
# cost = T.mean((network_output - target_values)**2)
# The loss function is calculated as the mean of the (categorical) cross-entropy between the prediction and target.
#! cost = T.nnet.categorical_crossentropy(network_output,target_var).mean()
cost = T.nnet.categorical_crossentropy(network_output,
target_values).mean()
# Retrieve all parameters from the network
all_params = lasagne.layers.get_all_params(l_out, trainable=True)
# Compute AdaGrad updates for training
writeLog("Computing updates...")
writeLog("Using optimizer: " + self.optimizer)
if (self.optimizer == "sgd"):
updates = lasagne.updates.sgd(cost, all_params, self.learning_rate)
elif (self.optimizer == "adagrad"):
updates = lasagne.updates.adagrad(cost, all_params,
self.learning_rate)
elif (self.optimizer == "adadelta"):
updates = lasagne.updates.adagrad(cost, all_params,
self.learning_rate, 0.95)
elif (self.optimizer == "momentum"):
updates = lasagne.updates.momentum(cost, all_params,
self.learning_rate, 0.9)
elif (self.optimizer == "nesterov_momentum"):
updates = lasagne.updates.nesterov_momentum(
cost, all_params, self.learning_rate, 0.9)
elif (self.optimizer == "rmsprop"):
updates = lasagne.updates.rmsprop(cost, all_params,
self.learning_rate, 0.9)
else:
updates = lasagne.updates.adam(cost,
all_params,
self.learning_rate,
beta1=0.9,
beta2=0.999)
# Theano functions for training and computing cost
writeLog("Compiling train function...")
self.train = theano.function(
[l_in.input_var, target_values, l_mask.input_var],
cost,
updates=updates,
allow_input_downcast=True)
#! self.train = theano.function([l_in.input_var, target_var, l_mask.input_var], cost, updates=updates, allow_input_downcast=True)
writeLog("Compiling train cost computing function...")
self.compute_cost = theano.function(
[l_in.input_var, target_values, l_mask.input_var],
cost,
allow_input_downcast=True)
#! self.compute_cost = theano.function([l_in.input_var, target_var, l_mask.input_var], cost, allow_input_downcast=True)
# In order to generate text from the network, we need the probability distribution of the next character given
# the state of the network and the input (a seed).
# In order to produce the probability distribution of the prediction, we compile a function called probs.
writeLog("Compiling propabilities computing function...")
self.propabilities = theano.function(
[l_in.input_var, l_mask.input_var],
network_output,
allow_input_downcast=True)
self.start_time = time.time()
self.previous_time = self.start_time
self.cumul_train_time = 0
self.cumul_test_time = 0
self.auc = 0
self.sr_trains = []
self.sr_tests = []
self.sr_tests_75p = []
self.sr_tests_50p = []
self.sr_tests_25p = []
self.sr_examplesSeen = []
self.time_used = []
self.avg_costs = []
self.time_used_for_test = []
self.all_cms = []
def predict_outcome(tracesToCalculateFor, selIndex, notSelIndex,
tracePercentage):
batches, masks = self.gen_prediction_data(tracesToCalculateFor,
tracePercentage)
correct = 0
predictions = []
probs_out = []
for i in range(len(batches)):
x = batches[i]
mask = masks[i]
probs = self.propabilities(x, mask)
for prob in enumerate(probs):
selProb = prob[1][selIndex]
notSelProb = prob[1][notSelIndex]
probs_out.append(selProb / (selProb + notSelProb))
predictions.append(selProb >= notSelProb)
return predictions, probs_out
def calculateSuccessRate(tracesToCalculateFor, tracePercentage,
testId):
selIndex = self.word_to_index[IN_SELECTION_TOKEN]
notSelIndex = self.word_to_index[NOT_IN_SELECTION_TOKEN]
predictions, probs = predict_outcome(tracesToCalculateFor,
selIndex, notSelIndex,
tracePercentage)
numSuccess = 0
cm = [0, 0, 0, 0]
exps = []
for i in range(len(tracesToCalculateFor)):
expected = tracesToCalculateFor[i].isSelected
actual = predictions[i]
exps.append(1 if expected else 0)
numSuccess += 1 if expected == actual else 0
cm[0] += 1 if expected and actual else 0
cm[1] += 1 if not expected and not actual else 0
cm[2] += 1 if not expected and actual else 0
cm[3] += 1 if expected and not actual else 0
self.cms[testId] = cm
self.cms_str += ":%i_%i_%i_%i" % (cm[0], cm[1], cm[2], cm[3])
if (testId == 1):
self.auc = metrics.roc_auc_score(exps, probs)
return numSuccess / len(tracesToCalculateFor)
def report(num_examples_seen, it, avg_cost, num_report_iterations):
t2 = time.time()
tutrain = (t2 - self.previous_time)
self.cumul_train_time = self.cumul_train_time + tutrain
self.time_used.append(tutrain)
self.generate_trace(5)
self.sr_examplesSeen.append(num_examples_seen)
self.cms = {}
self.cms_str = ""
writeLog("Testing 100% training samples")
sr_train = calculateSuccessRate(self.traces_train, 1.0, 0)
self.sr_trains.append(sr_train)
writeLog("Testing 100% test samples")
sr_test = calculateSuccessRate(self.traces_test, 1.0, 1)
writeLog("Testing 75% test samples")
sr_tests_75p = calculateSuccessRate(self.traces_test, 0.75, 2)
writeLog("Testing 50% test samples")
sr_tests_50p = calculateSuccessRate(self.traces_test, 0.5, 3)
writeLog("Testing 25% test samples")
sr_tests_25p = calculateSuccessRate(self.traces_test, 0.25, 4)
self.sr_tests.append(sr_test)
self.sr_tests_75p.append(sr_tests_75p)
self.sr_tests_50p.append(sr_tests_50p)
self.sr_tests_25p.append(sr_tests_25p)
self.avg_costs.append(avg_cost)
data_size = len(self.TS_train)
epoch = it * self.batch_size / data_size
t3 = time.time()
tutest = (t3 - t2)
self.cumul_test_time = self.cumul_test_time + tutest
self.previous_time = t3
self.time_used_for_test.append(tutest)
self.all_cms.append(self.cms)
writeLog(
"Iteration: %i (%i) Total time used: ~%f seconds (train: %f, test: %f)"
% (num_report_iterations, num_examples_seen,
(time.time() - self.start_time) * 1., self.cumul_train_time,
self.cumul_test_time))
writeLog("Epoch {} average loss = {}".format(epoch, avg_cost))
writeLog(
"Success rates: test: %f test 75%%: %f test 50%%: %f test 25%%: %f train: %f"
%
(sr_test, sr_tests_75p, sr_tests_50p, sr_tests_25p, sr_train))
writeResultRow([
datetime.now().replace(microsecond=0).isoformat(), "ok", "",
self.case_name, self.dataset_name, self.dataset_size,
self.algorithm, self.num_layers, self.hidden_dim_size,
self.optimizer, self.learning_rate, self.seq_length,
self.batch_size, self.grad_clipping,
self.num_iterations_between_reports, num_report_iterations,
num_examples_seen, epoch, tutrain, self.cumul_train_time,
tutest, self.cumul_test_time, sr_train, sr_test, sr_tests_75p,
sr_tests_50p, sr_tests_25p, avg_cost, self.auc, self.cms[1][0],
self.cms[1][1], self.cms[1][2], self.cms[1][3],
str(self.cms_str), self.predict_only_outcome,
self.final_trace_only, self.trace_length_modifier,
self.num_iterations_between_reports *
self.num_callbacks == 100000 * 50, self.max_num_words,
self.truncate_unknowns
])
# self.draw_chart()
# writeLog("Calculating initial probabilities.")
# self.sr_examplesSeen.append(0)
self.cms = {}
self.cms_str = ""
# sr_train = calculateSuccessRate(self.traces_train, 1.0, 0)
# self.sr_trains.append(sr_train)
# sr_test = calculateSuccessRate(self.traces_test, 1.0, 1)
# self.sr_tests.append(sr_test)
# self.time_used.append(time.time() - self.start_time)
# self.avg_costs.append(0)
# writeLog("Initial success rates: test: %f train: %f" % (sr_test, sr_train))
num_examples_seen = self.trainModel(report)
self.cms = {}
self.cms_str = ""
self.sr_examplesSeen.append(num_examples_seen)
sr_train = calculateSuccessRate(self.traces_train, 1.0, 0)
self.sr_trains.append(sr_train)
sr_test = calculateSuccessRate(self.traces_test, 1.0, 1)
self.sr_tests.append(sr_test)
self.avg_costs.append(0)
writeLog("Final success rates: test: %f train: %f" %
(sr_test, sr_train))
self.time_used.append(self.cumul_train_time)
def train(modelConfig, dataConfig, logConfig):
"""
训练
:param modelConfig: 模型配置
:param dataConfig: 数据配置
:param logConfig: 日志配置
:return:
"""
# 模型配置
model = modelConfig['model']
criterion = modelConfig['criterion']
optimzer = modelConfig['optimzer']
epochs = modelConfig['epochs']
device = modelConfig['device']
# 数据加载器
trainLoader = dataConfig['trainLoader']
validLoader = dataConfig['validLoader']
trainLength = dataConfig['trainLength']
validLength = dataConfig['validLength']
# 日志及模型保存
modelPath = logConfig['modelPath']
historyPath = logConfig['historyPath']
logPath = logConfig['logPath']
lastModelPath = logConfig['lastModelPath']
trainLosses = []
trainAcces = []
validLosses = []
validAcces = []
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
print('train is starting in ' + now)
best_L1_Acc = 0.
best_L2_Acc = 0.
best_L3_Acc = 0.
best_concat_Acc = 0.
best_com_Acc = 0.
best_epoch = 0
for epoch in range(epochs):
print("Epoch{}/{}".format(epoch, epochs))
print("-" * 10)
loss_1, loss_2, loss_3, loss_concat, loss, acc_1, acc_2, acc_3, acc_com \
= oneEpoch_train(model, trainLoader, optimzer, criterion, device)
val_loss_1, val_acc_1, val_loss_2, val_acc_2, val_loss_3, val_acc_3, val_loss_concat, val_acc_concat, val_loss_com, val_acc_com\
= oneEpoch_valid(model, validLoader, criterion, device)
loss_1 = loss_1 / len(trainLoader)
loss_2 = loss_2 / len(trainLoader)
loss_3 = loss_3 / len(trainLoader)
loss_concat = loss_concat / len(trainLoader)
loss = loss / len(trainLoader)
acc_1 = acc_1 / trainLength
acc_2 = acc_2 / trainLength
acc_3 = acc_3 / trainLength
acc_concat = acc_com / trainLength
val_loss_1 = val_loss_1 / len(validLoader)
val_loss_2 = val_loss_2 / len(validLoader)
val_loss_3 = val_loss_3 / len(validLoader)
val_loss_concat = val_loss_concat / len(validLoader)
val_loss_com = val_loss_com / len(validLoader)
val_acc_1 = val_acc_1 / validLength
val_acc_2 = val_acc_2 / validLength
val_acc_3 = val_acc_3 / validLength
val_acc_concat = val_acc_concat / validLength
val_acc_com = val_acc_com / validLength
# 模型验证有进步时,保存模型
if val_acc_1 > best_L1_Acc:
best_L1_Acc = val_acc_1
if val_acc_2 > best_L2_Acc:
best_L2_Acc = val_acc_2
if val_acc_3 > best_L3_Acc:
best_L3_Acc = val_acc_3
if val_acc_concat > best_concat_Acc:
best_concat_Acc = val_acc_concat
if val_acc_com > best_com_Acc:
best_epoch = epoch
best_com_Acc = val_acc_com
# saveModel(model,modelPath)
# 训练日志
now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
train_L1_Log = now + " Train L1 loss is :{:.4f},Train accuracy is:{:.4f}%\n".format(
loss_1, 100 * acc_1)
train_L2_Log = now + " Train L2 loss is :{:.4f},Train accuracy is:{:.4f}%\n".format(
loss_2, 100 * acc_2)
train_L3_Log = now + " Train L3 loss is :{:.4f},Train accuracy is:{:.4f}%\n".format(
loss_3, 100 * acc_3)
train_concat_Log = now + " Train concat loss is :{:.4f},Train accuracy is:{:.4f}%\n".format(
loss_concat, 100 * acc_concat)
train_total_Log = now + " Train total loss is :{:.4f}\n\n".format(loss)
val_L1_log = now + " Valid L1 loss is :{:.4f},Valid accuracy is:{:.4f}%\n".format(
val_loss_1, 100 * val_acc_1)
val_L2_log = now + " Valid L2 loss is :{:.4f},Valid accuracy is:{:.4f}%\n".format(
val_loss_2, 100 * val_acc_2)
val_L3_log = now + " Valid L3 loss is :{:.4f},Valid accuracy is:{:.4f}%\n".format(
val_loss_3, 100 * val_acc_3)
val_concat_log = now + " Valid concat loss is :{:.4f},Valid accuracy is:{:.4f}%\n".format(
val_loss_concat, 100 * val_acc_concat)
val_com_log = now + " Valid com loss is :{:.4f},Valid accuracy is:{:.4f}%\n\n".format(
val_loss_com, 100 * val_acc_com)
best_L1_log = now + ' best L1 Acc is {:.4f}%\n'.format(
100 * best_L1_Acc)
best_L2_log = now + ' best L2 Acc is {:.4f}%\n'.format(
100 * best_L2_Acc)
best_L3_log = now + ' best L3 Acc is {:.4f}%\n'.format(
100 * best_L3_Acc)
best_concat_log = now + ' best concat Acc is {:.4f}%\n'.format(
100 * best_concat_Acc)
best_com_log = now + ' best com Acc is {:.4f}%\n'.format(
100 * best_com_Acc)
best_epoch_log = now + ' best Acc epoch is :' + str(
best_epoch) + "\n\n"
train_log = train_L1_Log + train_L2_Log + train_L3_Log + train_concat_Log + train_total_Log
val_log = val_L1_log + val_L2_log + val_L3_log + val_concat_log + val_com_log
best_log = best_L1_log + best_L2_log + best_L3_log + best_concat_log + best_com_log + best_epoch_log
print(train_log + val_log + best_log)
# 训练历史 每个EPOCH都覆盖一次
# history = {
# 'trainLosses':trainLosses,
# 'trainAcces':trainAcces,
# 'validLosses':validLosses,
# 'validAcces':validAcces
# }
writeLog(logPath, train_log + val_log + best_log)
def run(self):
"""Run the main loop."""
while(True):
self._captureManager.enterFrame()
frame = self._captureManager.frame
if frame is not None:
if (self.autoBackgroundImg == None):
self.autoBackgroundImg = numpy.float32(frame)
rawFrame = frame.copy()
# First work out the region of interest by
# subtracting the fixed background image
# to create a mask.
#print frame
#print self._background_depth_img
absDiff = cv2.absdiff(frame,self._background_depth_img)
benMask,maskArea = filters.getBenMask(absDiff,8)
cv2.accumulateWeighted(frame,
self.autoBackgroundImg,0.05)
# Convert the background image into the same format
# as the main frame.
#bg = self.autoBackgroundImg
bg = cv2.convertScaleAbs(self.autoBackgroundImg,
alpha=1.0)
# Subtract the background from the frame image
cv2.absdiff(frame,bg,frame)
# Scale the difference image to make it more sensitive
# to changes.
cv2.convertScaleAbs(frame,frame,alpha=100)
# Apply the mask so we only see the test subject.
frame = cv2.multiply(frame,benMask,dst=frame,dtype=-1)
if (maskArea <= self.cfg.getConfigInt('area_threshold')):
bri=(0,0,0)
else:
# Calculate the brightness of the test subject.
bri = filters.getMean(frame,benMask)
# Add the brightness to the time series ready for analysis.
self._ts.addSamp(bri[0])
self._ts.addImg(rawFrame)
# Write timeseries to a file every 'output_framecount' frames.
if (self._outputFrameCount >= self.cfg.getConfigInt('output_framecount')):
# Write timeseries to file
self._ts.writeToFile("%s/%s" % \
( self.cfg.getConfigStr('output_directory'),
self.cfg.getConfigStr('ts_fname')
))
self._outputFrameCount = 0
else:
self._outputFrameCount = self._outputFrameCount + 1
# Only do the analysis every 15 frames (0.5 sec), or whatever
# is specified in configuration file analysis_framecount
# parameter.
if (self._frameCount < self.cfg.getConfigInt('analysis_framecount')):
self._frameCount = self._frameCount +1
else:
# Look for peaks in the brightness (=movement).
self._nPeaks,self._ts_time,self._rate = self._ts.findPeaks()
#print "%d peaks in %3.2f sec = %3.1f bpm" % \
# (nPeaks,ts_time,rate)
oldStatus = self._status
if (maskArea > self.cfg.getConfigInt('area_threshold')):
# Check for alarm levels
if (self._rate > self.cfg.getConfigInt(
"rate_warn")):
self._status= self.ALARM_STATUS_OK
elif (self._rate > self.cfg.getConfigInt(
"rate_alarm")):
self._status= self.ALARM_STATUS_WARN
else:
self._status= self.ALARM_STATUS_FULL
else:
self._status = self.ALARM_STATUS_NOT_FOUND
if (oldStatus == self.ALARM_STATUS_OK and
self._status == self.ALARM_STATUS_WARN) or \
(oldStatus == self.ALARM_STATUS_WARN and
self._status == self.ALARM_STATUS_FULL):
# Write timeseries to file
self._ts.writeToFile("%s/%s" % \
( self.cfg.getConfigStr('output_directory'),
self.cfg.getConfigStr('alarm_ts_fname')
),bgImg=self._background_depth_img)
# Collect the analysis results together and send them
# to the web server.
resultsDict = {}
resultsDict['fps'] = "%3.0f" % self.fps
resultsDict['bri'] = "%4.0f" % self._ts.mean
resultsDict['area'] = "%6.0f" % maskArea
resultsDict['nPeaks'] = "%d" % self._nPeaks
resultsDict['ts_time'] = self._ts_time
resultsDict['rate'] = "%d" % self._rate
resultsDict['time_t'] = time.ctime()
resultsDict['status'] = self._status
self._ws.setAnalysisResults(resultsDict)
# Write the results to file as a json string
utils.writeJSON(resultsDict,"%s/%s" % \
(self._tmpdir,
self.cfg.getConfigStr("data_fname")))
utils.writeLog(resultsDict,"%s/%s" % \
(self._tmpdir,
"benFinder_alarms.log"))
# Plot the graph of brightness, and save the images
# to disk.
self._ts.plotRawData(
file=True,
fname="%s/%s" % \
(self._tmpdir,self.cfg.getConfigStr("chart_fname")))
cv2.imwrite("%s/%s" % (self._tmpdir,
self.cfg.getConfigStr(
"raw_image_fname")),
rawFrame)
cv2.imwrite("%s/%s" % (self._tmpdir,self.cfg.getConfigStr(
"masked_image_fname")),
frame)
self._frameCount = 0
else:
print "Null frame received - assuming end of file and exiting"
break
self._captureManager.exitFrame()
