def __init__(self):
self.model = load_model(MODEL_PATH)
self.anomalyModel = AnomalyDetector()
self.anomalyModel.load(ANOMALY_MODEL_PATH)
self.anomalyModel.load_inv(ANOMALY_INV_MODEL_PATH)
with open(DATA_GEN_PATH, 'rb') as dgfile:
self.datagen = pickle.load(dgfile, encoding='latin1')
# self.datagen = ImageDataGenerator(
# zca_whitening=True,
# )
with open(CLASS_LIST, 'r') as clfile:
self.classList = json.load(clfile)
with open(TEXT_CLASS_LIST) as f:
text_class = json.load(f)
self.text_id_map = {}
for k, v in text_class.items():
for t in v:
self.text_id_map[t] = k
with open(WIDGET_CLASS_LIST) as f:
self.widgetClassList = json.load(f)
self.x = None
pass
def __init__(self,
model_path=None,
class_names=None,
datagen=None,
anomaly=True,
input_size=32):
if not model_path:
model_path = self.default_settings['model_path']
if not class_names:
with open(self.default_settings['validation_metadata']
) as class_names_file:
class_names = json.load(class_names_file)['class_names']
if not datagen:
with open(self.default_settings['datagen'], 'rb') as datagen_file:
datagen = pickle.load(datagen_file)
self.anomaly_model = None
if anomaly:
self.anomaly_model = AnomalyDetector()
self.anomaly_model.load(self.default_settings['anomaly_model'])
self.anomaly_model.load_inv(
self.default_settings['inv_anomaly_model'])
self.model = load_model(model_path)
self.class_names = class_names
self.datagen = datagen
self.input_size = input_size
self.embeddings_model = None
def modiPath():
host_collector_address = base_address + '\\CompleteModel\\HuaJiao_30_host_collector_modipath.pickle'
whole_result_address = base_address + '\\Path_Modi\\result_huajiao_wvs_1113_modipath_modi'
path_para_dic_address = base_address + '\\Path_Modi\\modi_path_dict.pickle'
path_collector = []
with open(host_collector_address, 'rb') as read_host_collector:
host_collector = pickle.load(read_host_collector)
anomaly_detector = AnomalyDetector()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
for line in infile:
line = line.strip(' \n')
record = line.split('\t')
if len(record) == 13 and record[_host] != '':
a_record = UrlRecord(record)
current_model = host_collector.get_host_model(
a_record.get_host())
pattern_flag = current_model.getDetectFlag()
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
print 'Study consuming: %s' % study_interval
anomaly_status = anomaly_detector.detectpath(
a_record, current_model, 'new')
path_collector.append(anomaly_status['path_para'])
if anomaly_status['Result'] is True:
anomaly_writer.writePathResult(
a_record, anomaly_status)
record_num += 1
if record_num % 1000 == 0:
print 'Record completed: ', record_num,
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % (
current_model.getUrlAmount(),
current_model.getDifUrlAmount(),
current_model.getSipAmount())
except IndexError:
print 'Lancer says:\n' \
'\tIt has error through the system.'
raise
with open(path_para_dic_address, 'wb') as save:
pickle.dump(path_collector, save)
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
def train(source, packetnum):
try:
fe = createFE(source, packetnum)
anomDetector = AnomalyDetector(trainOnNum=packetnum)
while not anomDetector.isTrained():
if fe.reachedPacketLimit():
raise RuntimeError('Not enough packets for training.')
process_packet(fe, anomDetector)
except KeyboardInterrupt:
print("\nKeyboard interrupt.\n")
finally:
serializeModel(anomDetector)
def modiPath():
host_collector_address = base_address + '\\CompleteModel\\HuaJiao_30_host_collector_modipath.pickle'
whole_result_address = base_address + '\\Path_Modi\\result_huajiao_wvs_1113_modipath_modi'
path_para_dic_address = base_address + '\\Path_Modi\\modi_path_dict.pickle'
path_collector = []
with open(host_collector_address, 'rb') as read_host_collector:
host_collector = pickle.load(read_host_collector)
anomaly_detector = AnomalyDetector()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
for line in infile:
line = line.strip(' \n')
record = line.split('\t')
if len(record) == 13 and record[_host] != '':
a_record = UrlRecord(record)
current_model = host_collector.get_host_model(a_record.get_host())
pattern_flag = current_model.getDetectFlag()
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
print 'Study consuming: %s' % study_interval
anomaly_status = anomaly_detector.detectpath(a_record, current_model, 'new')
path_collector.append(anomaly_status['path_para'])
if anomaly_status['Result'] is True:
anomaly_writer.writePathResult(a_record, anomaly_status)
record_num += 1
if record_num % 1000 == 0:
print 'Record completed: ', record_num,
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount())
except IndexError:
print 'Lancer says:\n' \
'\tIt has error through the system.'
raise
with open(path_para_dic_address, 'wb') as save:
pickle.dump(path_collector, save)
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
def main():
with open(host_collector_address, 'rb') as read_host_collector:
host_collector = pickle.load(read_host_collector)
anomaly_detector = AnomalyDetector()
anomaly_collector = AnomalyCollector()
result_controller = ResultController()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
for line in infile:
line = line.strip(' \n')
record = line.split('\t')
if len(record) == 13 and record[_host] != '':
a_record = UrlRecord(record)
current_model = host_collector.get_host_model(a_record.get_host())
pattern_flag = current_model.getDetectFlag()
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
print 'Study consuming: %s' % study_interval
anomaly_status = anomaly_detector.detect(a_record, current_model)
if anomaly_status['Result'] is True:
valid_flag = result_controller.isValid(a_record, anomaly_collector)
if valid_flag is True:
anomaly_writer.writeCompleteResult(a_record, anomaly_status)
# anomaly_writer.writeTimeAttribute(a_record)
else:
current_model.reStudy()
record_num += 1
if record_num % 1000 == 0:
print 'Record completed: ', record_num,
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount())
except IndexError:
print 'Lancer says:\n' \
'\tIt has error through the system.'
raise
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
def __init__(self, device_name, buffer_size=100, upload_to_iotb=False):
global plot_this
self.device_id = device_name
self.anomaly_detector = AnomalyDetector(
buffer_size, window_length_sec=10, num_windows=50, plot_time=False, plot=plot_this
)
plot_this = False
self.upload_to_iotb = upload_to_iotb
self.pkts_since_upload = []
self.server_on = False
try:
r = requests.get(SERVER + "?device_id=d1")
self.server_on = True
except requests.exceptions.ConnectionError:
self.server_on = False
class DlInferenceService(object):
default_settings = {
'model_path': './saved_models/small_cnn_weights_50_512.h5',
'validation_metadata': './saved_models/validation_metadata.json',
'datagen': './saved_models/datagen.pkl',
'anomaly_model': './saved_models/anomaly.pkl',
'inv_anomaly_model': './saved_models/inv_anomaly.pkl',
}
def __init__(self,
model_path=None,
class_names=None,
datagen=None,
anomaly=True,
input_size=32):
if not model_path:
model_path = self.default_settings['model_path']
if not class_names:
with open(self.default_settings['validation_metadata']
) as class_names_file:
class_names = json.load(class_names_file)['class_names']
if not datagen:
with open(self.default_settings['datagen'], 'rb') as datagen_file:
datagen = pickle.load(datagen_file)
self.anomaly_model = None
if anomaly:
self.anomaly_model = AnomalyDetector()
self.anomaly_model.load(self.default_settings['anomaly_model'])
self.anomaly_model.load_inv(
self.default_settings['inv_anomaly_model'])
self.model = load_model(model_path)
self.class_names = class_names
self.datagen = datagen
self.input_size = input_size
self.embeddings_model = None
def _load_image(self, image_path):
image = self._preprocess_image(Image.open(image_path))
return self._image_to_numpy_array(image)
def _preprocess_image(self, image):
size = (self.input_size, self.input_size)
return image.convert('L').resize(size, Image.ANTIALIAS)
def _image_to_numpy_array(self, image):
size = (self.input_size, self.input_size, 1)
output = np.asarray(image, dtype='int32')
output = output.astype('float32')
output /= 255
return output.reshape(size)
def _image_paths_to_numpy_array(self, image_paths):
image_count = len(image_paths)
x_shape = (image_count, self.input_size, self.input_size, 1)
x = np.zeros(x_shape)
for idx, image_path in enumerate(image_paths):
x[idx] = self._load_image(image_path)
return x
def _initialize_datagen(self):
datagen = ImageDataGenerator(zca_whitening=True, )
return datagen
def _predict(self, x):
x = self.datagen.flow(x, batch_size=len(x), shuffle=False).next()
result = self.model.predict(x)
return result
def _predictions_to_class(self, result):
predicted_class = np.argmax(result, axis=1)
return predicted_class
def _initialize_embedding_model(self, embedding_layer_name='embedding'):
self.embedding_model = Sequential()
for layer in self.model.layers:
self.embedding_model.add(layer)
if layer.name == embedding_layer_name:
break
def identify_anomalies(self, x):
pass
def generate_embeddings(self, x):
if not self.embeddings_model:
self._initialize_embedding_model()
x = self.datagen.flow(x, batch_size=len(x), shuffle=False).next()
embeddings = self.embedding_model.predict(x)
return embeddings
def classify(self, x):
predictions = self._predict(x)
print(predictions.shape)
anomalies = np.zeros(len(predictions))
if self.anomaly_model:
anomalies = self.anomaly_model.predict(predictions)
return [
self.class_names[prediction] if not anomalies[idx] else 'anomaly'
for idx, prediction in enumerate(
self._predictions_to_class(predictions))
]
def classify_paths(self, image_paths):
x = self._image_paths_to_numpy_array(image_paths)
return self.classify(x)
def classify_images(self, images):
image_count = len(images)
x_shape = (image_count, self.input_size, self.input_size, 1)
x = np.zeros(x_shape)
for idx, image in enumerate(images):
x[idx] = self._image_to_numpy_array(self._preprocess_image(image))
return self.classify(x)
def create_npy(self, image_paths, out_path):
x = self._image_paths_to_numpy_array(image_paths)
np.save(out_path, x)
def make_preprocessed_images(self, npy, augment=False):
x = np.load(npy)
if augment:
x = self.datagen.flow(x, batch_size=10, shuffle=False).next()
for idx, row in enumerate(x):
print(row.shape)
bla = Image.fromarray(row.reshape(32, 32) * 255).convert('RGB')
out_dir = '{}_preprocessed{}'.format(
npy, '_augmented' if augment else '')
if not os.path.exists(out_dir):
os.mkdir(out_dir)
bla.save('{}/{}.png'.format(out_dir, idx))
from Evaluation import Evaluation
from StableFrameList import StableFrameList
from MaskList import MaskList
from Misc import Image
from AnomalyDetector import AnomalyDetector
import ResultRefinement as rr
#Initilize detector
print('Parse detector result ...')
dayNightDetector = DayNightDetector()
detectorDay = DetectorDay(Config.data_path + '/result_8_3_3_clas.txt',
Config.data_path + '/result_8_3_3_nclas.txt')
detectorNight = DetectorNight(Config.data_path +
'/extracted-bboxes-dark-videos')
#evalFunc = Evaluation(Config.data_path + '/test_groundtruth.txt')
anomalyDetector = AnomalyDetector()
stableList = StableFrameList(Config.data_path +
'/unchanged_scene_periods.json')
maskList = MaskList(Config.data_path + '/masks_refine_v3')
for video_id in range(1, 101):
print("Processing video ", video_id)
detector = detectorDay
if dayNightDetector.checkNight(video_id):
detector = detectorNight
stableIntervals = stableList[video_id]
print(stableIntervals)
confs = {}
print(detector.name)
def main():
host_collector = HostCollector()
anomaly_detector = AnomalyDetector()
anomaly_collector = AnomalyCollector()
result_controller = ResultController()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
study_record_num = 0
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
a_url = UrlRecord(record)
current_model = host_collector.getHostModel(
a_url.get_host())
pattern_flag = current_model.getDetectFlag()
# Judge Host-model's pattern('Study ready' or 'Study...')
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
study_record_num = record_num
print 'Study consuming: %s' % study_interval
print 'Study Record: %s' % study_record_num
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
(current_model.getUrlAmount(), current_model.getDifUrlAmount(),
current_model.getSipAmount())
# Detect whether the record is anomaly
anomaly_status = anomaly_detector.detect(
a_url, current_model)
# If the record is detected to be anomaly
if anomaly_status['Result'] is True:
valid_flag = result_controller.isValid(
a_url, anomaly_collector)
if valid_flag is True:
anomaly_writer.writeResult(
a_url, anomaly_status)
else:
current_model.reStudy()
elif pattern_flag == 'Study...':
current_model.add_record(a_url)
else:
raise ValueError(
'Lancer says: pattern selection error.')
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: ', record_num,
else:
detect_record_num = record_num - study_record_num
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print 'Detect Record: %s' % detect_record_num
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
# Store the completed Host Model.
with open(host_collector_address, 'wb') as save_host_collector:
pickle.dump(host_collector, save_host_collector)
except IndexError:
print 'Lancer says:\n\tIt has error through the system.'
raise
class IconDetector:
def __init__(self):
self.model = load_model(MODEL_PATH)
self.anomalyModel = AnomalyDetector()
self.anomalyModel.load(ANOMALY_MODEL_PATH)
self.anomalyModel.load_inv(ANOMALY_INV_MODEL_PATH)
with open(DATA_GEN_PATH, 'rb') as dgfile:
self.datagen = pickle.load(dgfile, encoding='latin1')
# self.datagen = ImageDataGenerator(
# zca_whitening=True,
# )
with open(CLASS_LIST, 'r') as clfile:
self.classList = json.load(clfile)
with open(TEXT_CLASS_LIST) as f:
text_class = json.load(f)
self.text_id_map = {}
for k, v in text_class.items():
for t in v:
self.text_id_map[t] = k
with open(WIDGET_CLASS_LIST) as f:
self.widgetClassList = json.load(f)
self.x = None
pass
def preprocessImgpath(self, imgpath):
image = Image.open(imgpath)
self.preprocess(image)
pass
def preprocess(self, image):
imageSize = (INPUT_SIZE, INPUT_SIZE)
image = image.convert('L').resize(imageSize, Image.ANTIALIAS)
singleArrSize = (INPUT_SIZE, INPUT_SIZE, 1)
image = np.asarray(image, dtype='int32')
image = image.astype('float32')
image /= 255
image = image.reshape(singleArrSize)
xShape = (1, INPUT_SIZE, INPUT_SIZE, 1)
x = np.zeros(xShape)
x[0] = image
self.x = x
pass
def predict(self):
x = self.datagen.flow(self.x, batch_size=1, shuffle=False).next()
# preImg = Image.fromarray(x.reshape(32, 32) * 255).convert('RGB')
# preImg.save("./n.png")
prediction = self.model.predict(x)
anomalies = np.zeros(1)
if self.anomalyModel:
anomalies = self.anomalyModel.predict(prediction)
predictClass = np.argmax(prediction, axis=1)
if anomalies[0]:
print("[INFO] predict result : anomaly")
return 99 # 其它图标
else:
print("[INFO] predict result : " +
self.classList[str(predictClass[0])])
return predictClass[0]
pass
def main():
host_collector = HostCollector()
anomaly_detector = AnomalyDetector()
anomaly_collector = AnomalyCollector()
result_controller = ResultController()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
study_record_num = 0
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
a_url = UrlRecord(record)
current_model = host_collector.getHostModel(a_url.get_host())
pattern_flag = current_model.getDetectFlag()
# Judge Host-model's pattern('Study ready' or 'Study...')
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
study_record_num = record_num
print 'Study consuming: %s' % study_interval
print 'Study Record: %s' % study_record_num
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
(current_model.getUrlAmount(), current_model.getDifUrlAmount(),
current_model.getSipAmount())
# Detect whether the record is anomaly
anomaly_status = anomaly_detector.detect(a_url, current_model)
# If the record is detected to be anomaly
if anomaly_status['Result'] is True:
valid_flag = result_controller.isValid(a_url, anomaly_collector)
if valid_flag is True:
anomaly_writer.writeResult(a_url, anomaly_status)
else:
current_model.reStudy()
elif pattern_flag == 'Study...':
current_model.add_record(a_url)
else:
raise ValueError('Lancer says: pattern selection error.')
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: ', record_num,
else:
detect_record_num = record_num - study_record_num
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print 'Detect Record: %s' % detect_record_num
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
# Store the completed Host Model.
with open(host_collector_address, 'wb') as save_host_collector:
pickle.dump(host_collector, save_host_collector)
except IndexError:
print 'Lancer says:\n\tIt has error through the system.'
raise
class IOTdevice:
def __init__(self, device_name, buffer_size=100, upload_to_iotb=False):
global plot_this
self.device_id = device_name
self.anomaly_detector = AnomalyDetector(
buffer_size, window_length_sec=10, num_windows=50, plot_time=False, plot=plot_this
)
plot_this = False
self.upload_to_iotb = upload_to_iotb
self.pkts_since_upload = []
self.server_on = False
try:
r = requests.get(SERVER + "?device_id=d1")
self.server_on = True
except requests.exceptions.ConnectionError:
self.server_on = False
# self.window_anomaly_detector = WindowAnomalyDetector(window_length_sec=20, num_windows=30, ML_alg=sklearn.svm.OneClassSVM)
def build_model(self, pf):
"""build database of pktFeatures. Upload every 10 packets. Featurizer is non json so must convert to dict"""
get_pkt_type = {
ICMP: "ICMP",
DNS: "DNS",
STP: "STP",
DHCP: "DHCP",
TCP: "TCP",
UDP: "UDP",
IP: "IP",
ARP: "ARP",
Ether: "Ether",
}
features = {}
features["arrival_time"] = pf.arrival_time
features["len_bytes"] = pf.len_bytes
features["pkt_type"] = get_pkt_type[pf.pkt_type]
features["features"] = pf.features
self.pkts_since_upload.append(features)
if len(self.pkts_since_upload) >= pkts_per_upload:
self.upload_model()
def upload_model(self):
"""upload features to SERVER"""
payload = {"device_id": self.device_id, "data": self.pkts_since_upload}
r = requests.post(SERVER, json=payload)
# print payload
def download_model(self):
"""download model if device was identified"""
if self.server_on:
r = requests.get(SERVER + "?device_id=" + self.device_id)
# if model found on server
if r.status_code == 200:
try:
data = pickle.loads(r.content)
for each_pkt in data:
self.anomaly_detector.add_training_data(Featurizer(each_pkt))
except:
# print "Model for device {} not found on server".format(self.device_id)
pass
def handle_pkt(self, pkt):
pf = PktFeaturizer(pkt)
if self.server_on:
self.build_model(pf)
prediction, distance = self.anomaly_detector.predict(pf)
self.anomaly_detector.add_training_data(pf)
# self.window_anomaly_detector.predict(pf)
# self.window_anomaly_detector.add_training_data(pf)
return prediction
import pandas as pd
from AnomalyDetector import AnomalyDetector
from datetime import datetime
g = pd.read_csv('../data/gageids.csv', header=None, dtype='str')
ad = AnomalyDetector('streamanomaly', datetime.now())
ad.getAnomalies(g[0].values)