def __init__(self): self.model = load_model(MODEL_PATH) self.anomalyModel = AnomalyDetector() self.anomalyModel.load(ANOMALY_MODEL_PATH) self.anomalyModel.load_inv(ANOMALY_INV_MODEL_PATH) with open(DATA_GEN_PATH, 'rb') as dgfile: self.datagen = pickle.load(dgfile, encoding='latin1') # self.datagen = ImageDataGenerator( # zca_whitening=True, # ) with open(CLASS_LIST, 'r') as clfile: self.classList = json.load(clfile) with open(TEXT_CLASS_LIST) as f: text_class = json.load(f) self.text_id_map = {} for k, v in text_class.items(): for t in v: self.text_id_map[t] = k with open(WIDGET_CLASS_LIST) as f: self.widgetClassList = json.load(f) self.x = None pass
def __init__(self, model_path=None, class_names=None, datagen=None, anomaly=True, input_size=32): if not model_path: model_path = self.default_settings['model_path'] if not class_names: with open(self.default_settings['validation_metadata'] ) as class_names_file: class_names = json.load(class_names_file)['class_names'] if not datagen: with open(self.default_settings['datagen'], 'rb') as datagen_file: datagen = pickle.load(datagen_file) self.anomaly_model = None if anomaly: self.anomaly_model = AnomalyDetector() self.anomaly_model.load(self.default_settings['anomaly_model']) self.anomaly_model.load_inv( self.default_settings['inv_anomaly_model']) self.model = load_model(model_path) self.class_names = class_names self.datagen = datagen self.input_size = input_size self.embeddings_model = None
def modiPath(): host_collector_address = base_address + '\\CompleteModel\\HuaJiao_30_host_collector_modipath.pickle' whole_result_address = base_address + '\\Path_Modi\\result_huajiao_wvs_1113_modipath_modi' path_para_dic_address = base_address + '\\Path_Modi\\modi_path_dict.pickle' path_collector = [] with open(host_collector_address, 'rb') as read_host_collector: host_collector = pickle.load(read_host_collector) anomaly_detector = AnomalyDetector() anomaly_writer = AnomalyWriter(whole_result_address) try: # save the begin time begin_time = datetime.datetime.now() detect_time_flag = True global study_ready_time global detect_ready_time with open(url_address, 'rb') as infile: record_num = 0 for line in infile: line = line.strip(' \n') record = line.split('\t') if len(record) == 13 and record[_host] != '': a_record = UrlRecord(record) current_model = host_collector.get_host_model( a_record.get_host()) pattern_flag = current_model.getDetectFlag() if pattern_flag == 'Study ready': if detect_time_flag: detect_time_flag = False study_ready_time = datetime.datetime.now() study_interval = study_ready_time - begin_time print 'Study consuming: %s' % study_interval anomaly_status = anomaly_detector.detectpath( a_record, current_model, 'new') path_collector.append(anomaly_status['path_para']) if anomaly_status['Result'] is True: anomaly_writer.writePathResult( a_record, anomaly_status) record_num += 1 if record_num % 1000 == 0: print 'Record completed: ', record_num, print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % ( current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount()) except IndexError: print 'Lancer says:\n' \ '\tIt has error through the system.' raise with open(path_para_dic_address, 'wb') as save: pickle.dump(path_collector, save) end_time = datetime.datetime.now() detect_ready_time = end_time - study_ready_time print "Detection consuming: %s" % detect_ready_time print 'Whole system test finished.'
def train(source, packetnum): try: fe = createFE(source, packetnum) anomDetector = AnomalyDetector(trainOnNum=packetnum) while not anomDetector.isTrained(): if fe.reachedPacketLimit(): raise RuntimeError('Not enough packets for training.') process_packet(fe, anomDetector) except KeyboardInterrupt: print("\nKeyboard interrupt.\n") finally: serializeModel(anomDetector)
def modiPath(): host_collector_address = base_address + '\\CompleteModel\\HuaJiao_30_host_collector_modipath.pickle' whole_result_address = base_address + '\\Path_Modi\\result_huajiao_wvs_1113_modipath_modi' path_para_dic_address = base_address + '\\Path_Modi\\modi_path_dict.pickle' path_collector = [] with open(host_collector_address, 'rb') as read_host_collector: host_collector = pickle.load(read_host_collector) anomaly_detector = AnomalyDetector() anomaly_writer = AnomalyWriter(whole_result_address) try: # save the begin time begin_time = datetime.datetime.now() detect_time_flag = True global study_ready_time global detect_ready_time with open(url_address, 'rb') as infile: record_num = 0 for line in infile: line = line.strip(' \n') record = line.split('\t') if len(record) == 13 and record[_host] != '': a_record = UrlRecord(record) current_model = host_collector.get_host_model(a_record.get_host()) pattern_flag = current_model.getDetectFlag() if pattern_flag == 'Study ready': if detect_time_flag: detect_time_flag = False study_ready_time = datetime.datetime.now() study_interval = study_ready_time - begin_time print 'Study consuming: %s' % study_interval anomaly_status = anomaly_detector.detectpath(a_record, current_model, 'new') path_collector.append(anomaly_status['path_para']) if anomaly_status['Result'] is True: anomaly_writer.writePathResult(a_record, anomaly_status) record_num += 1 if record_num % 1000 == 0: print 'Record completed: ', record_num, print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount()) except IndexError: print 'Lancer says:\n' \ '\tIt has error through the system.' raise with open(path_para_dic_address, 'wb') as save: pickle.dump(path_collector, save) end_time = datetime.datetime.now() detect_ready_time = end_time - study_ready_time print "Detection consuming: %s" % detect_ready_time print 'Whole system test finished.'
def main(): with open(host_collector_address, 'rb') as read_host_collector: host_collector = pickle.load(read_host_collector) anomaly_detector = AnomalyDetector() anomaly_collector = AnomalyCollector() result_controller = ResultController() anomaly_writer = AnomalyWriter(whole_result_address) try: # save the begin time begin_time = datetime.datetime.now() detect_time_flag = True global study_ready_time global detect_ready_time with open(url_address, 'rb') as infile: record_num = 0 for line in infile: line = line.strip(' \n') record = line.split('\t') if len(record) == 13 and record[_host] != '': a_record = UrlRecord(record) current_model = host_collector.get_host_model(a_record.get_host()) pattern_flag = current_model.getDetectFlag() if pattern_flag == 'Study ready': if detect_time_flag: detect_time_flag = False study_ready_time = datetime.datetime.now() study_interval = study_ready_time - begin_time print 'Study consuming: %s' % study_interval anomaly_status = anomaly_detector.detect(a_record, current_model) if anomaly_status['Result'] is True: valid_flag = result_controller.isValid(a_record, anomaly_collector) if valid_flag is True: anomaly_writer.writeCompleteResult(a_record, anomaly_status) # anomaly_writer.writeTimeAttribute(a_record) else: current_model.reStudy() record_num += 1 if record_num % 1000 == 0: print 'Record completed: ', record_num, print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount()) except IndexError: print 'Lancer says:\n' \ '\tIt has error through the system.' raise end_time = datetime.datetime.now() detect_ready_time = end_time - study_ready_time print "Detection consuming: %s" % detect_ready_time print 'Whole system test finished.'
def __init__(self, device_name, buffer_size=100, upload_to_iotb=False): global plot_this self.device_id = device_name self.anomaly_detector = AnomalyDetector( buffer_size, window_length_sec=10, num_windows=50, plot_time=False, plot=plot_this ) plot_this = False self.upload_to_iotb = upload_to_iotb self.pkts_since_upload = [] self.server_on = False try: r = requests.get(SERVER + "?device_id=d1") self.server_on = True except requests.exceptions.ConnectionError: self.server_on = False
class DlInferenceService(object): default_settings = { 'model_path': './saved_models/small_cnn_weights_50_512.h5', 'validation_metadata': './saved_models/validation_metadata.json', 'datagen': './saved_models/datagen.pkl', 'anomaly_model': './saved_models/anomaly.pkl', 'inv_anomaly_model': './saved_models/inv_anomaly.pkl', } def __init__(self, model_path=None, class_names=None, datagen=None, anomaly=True, input_size=32): if not model_path: model_path = self.default_settings['model_path'] if not class_names: with open(self.default_settings['validation_metadata'] ) as class_names_file: class_names = json.load(class_names_file)['class_names'] if not datagen: with open(self.default_settings['datagen'], 'rb') as datagen_file: datagen = pickle.load(datagen_file) self.anomaly_model = None if anomaly: self.anomaly_model = AnomalyDetector() self.anomaly_model.load(self.default_settings['anomaly_model']) self.anomaly_model.load_inv( self.default_settings['inv_anomaly_model']) self.model = load_model(model_path) self.class_names = class_names self.datagen = datagen self.input_size = input_size self.embeddings_model = None def _load_image(self, image_path): image = self._preprocess_image(Image.open(image_path)) return self._image_to_numpy_array(image) def _preprocess_image(self, image): size = (self.input_size, self.input_size) return image.convert('L').resize(size, Image.ANTIALIAS) def _image_to_numpy_array(self, image): size = (self.input_size, self.input_size, 1) output = np.asarray(image, dtype='int32') output = output.astype('float32') output /= 255 return output.reshape(size) def _image_paths_to_numpy_array(self, image_paths): image_count = len(image_paths) x_shape = (image_count, self.input_size, self.input_size, 1) x = np.zeros(x_shape) for idx, image_path in enumerate(image_paths): x[idx] = self._load_image(image_path) return x def _initialize_datagen(self): datagen = ImageDataGenerator(zca_whitening=True, ) return datagen def _predict(self, x): x = self.datagen.flow(x, batch_size=len(x), shuffle=False).next() result = self.model.predict(x) return result def _predictions_to_class(self, result): predicted_class = np.argmax(result, axis=1) return predicted_class def _initialize_embedding_model(self, embedding_layer_name='embedding'): self.embedding_model = Sequential() for layer in self.model.layers: self.embedding_model.add(layer) if layer.name == embedding_layer_name: break def identify_anomalies(self, x): pass def generate_embeddings(self, x): if not self.embeddings_model: self._initialize_embedding_model() x = self.datagen.flow(x, batch_size=len(x), shuffle=False).next() embeddings = self.embedding_model.predict(x) return embeddings def classify(self, x): predictions = self._predict(x) print(predictions.shape) anomalies = np.zeros(len(predictions)) if self.anomaly_model: anomalies = self.anomaly_model.predict(predictions) return [ self.class_names[prediction] if not anomalies[idx] else 'anomaly' for idx, prediction in enumerate( self._predictions_to_class(predictions)) ] def classify_paths(self, image_paths): x = self._image_paths_to_numpy_array(image_paths) return self.classify(x) def classify_images(self, images): image_count = len(images) x_shape = (image_count, self.input_size, self.input_size, 1) x = np.zeros(x_shape) for idx, image in enumerate(images): x[idx] = self._image_to_numpy_array(self._preprocess_image(image)) return self.classify(x) def create_npy(self, image_paths, out_path): x = self._image_paths_to_numpy_array(image_paths) np.save(out_path, x) def make_preprocessed_images(self, npy, augment=False): x = np.load(npy) if augment: x = self.datagen.flow(x, batch_size=10, shuffle=False).next() for idx, row in enumerate(x): print(row.shape) bla = Image.fromarray(row.reshape(32, 32) * 255).convert('RGB') out_dir = '{}_preprocessed{}'.format( npy, '_augmented' if augment else '') if not os.path.exists(out_dir): os.mkdir(out_dir) bla.save('{}/{}.png'.format(out_dir, idx))
from Evaluation import Evaluation from StableFrameList import StableFrameList from MaskList import MaskList from Misc import Image from AnomalyDetector import AnomalyDetector import ResultRefinement as rr #Initilize detector print('Parse detector result ...') dayNightDetector = DayNightDetector() detectorDay = DetectorDay(Config.data_path + '/result_8_3_3_clas.txt', Config.data_path + '/result_8_3_3_nclas.txt') detectorNight = DetectorNight(Config.data_path + '/extracted-bboxes-dark-videos') #evalFunc = Evaluation(Config.data_path + '/test_groundtruth.txt') anomalyDetector = AnomalyDetector() stableList = StableFrameList(Config.data_path + '/unchanged_scene_periods.json') maskList = MaskList(Config.data_path + '/masks_refine_v3') for video_id in range(1, 101): print("Processing video ", video_id) detector = detectorDay if dayNightDetector.checkNight(video_id): detector = detectorNight stableIntervals = stableList[video_id] print(stableIntervals) confs = {} print(detector.name)
def main(): host_collector = HostCollector() anomaly_detector = AnomalyDetector() anomaly_collector = AnomalyCollector() result_controller = ResultController() anomaly_writer = AnomalyWriter(whole_result_address) try: # save the begin time begin_time = datetime.datetime.now() detect_time_flag = True global study_ready_time global detect_ready_time with open(url_address, 'rb') as infile: record_num = 0 study_record_num = 0 for line in infile: record = line.strip(' \n').split('\t') if len(record) == 13 and record[_host] != '': a_url = UrlRecord(record) current_model = host_collector.getHostModel( a_url.get_host()) pattern_flag = current_model.getDetectFlag() # Judge Host-model's pattern('Study ready' or 'Study...') if pattern_flag == 'Study ready': if detect_time_flag: detect_time_flag = False study_ready_time = datetime.datetime.now() study_interval = study_ready_time - begin_time study_record_num = record_num print 'Study consuming: %s' % study_interval print 'Study Record: %s' % study_record_num print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \ (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount()) # Detect whether the record is anomaly anomaly_status = anomaly_detector.detect( a_url, current_model) # If the record is detected to be anomaly if anomaly_status['Result'] is True: valid_flag = result_controller.isValid( a_url, anomaly_collector) if valid_flag is True: anomaly_writer.writeResult( a_url, anomaly_status) else: current_model.reStudy() elif pattern_flag == 'Study...': current_model.add_record(a_url) else: raise ValueError( 'Lancer says: pattern selection error.') record_num += 1 if record_num % 10000 == 0: print 'Record completed: ', record_num, else: detect_record_num = record_num - study_record_num end_time = datetime.datetime.now() detect_ready_time = end_time - study_ready_time print 'Detect Record: %s' % detect_record_num print "Detection consuming: %s" % detect_ready_time print 'Whole system test finished.' # Store the completed Host Model. with open(host_collector_address, 'wb') as save_host_collector: pickle.dump(host_collector, save_host_collector) except IndexError: print 'Lancer says:\n\tIt has error through the system.' raise
class IconDetector: def __init__(self): self.model = load_model(MODEL_PATH) self.anomalyModel = AnomalyDetector() self.anomalyModel.load(ANOMALY_MODEL_PATH) self.anomalyModel.load_inv(ANOMALY_INV_MODEL_PATH) with open(DATA_GEN_PATH, 'rb') as dgfile: self.datagen = pickle.load(dgfile, encoding='latin1') # self.datagen = ImageDataGenerator( # zca_whitening=True, # ) with open(CLASS_LIST, 'r') as clfile: self.classList = json.load(clfile) with open(TEXT_CLASS_LIST) as f: text_class = json.load(f) self.text_id_map = {} for k, v in text_class.items(): for t in v: self.text_id_map[t] = k with open(WIDGET_CLASS_LIST) as f: self.widgetClassList = json.load(f) self.x = None pass def preprocessImgpath(self, imgpath): image = Image.open(imgpath) self.preprocess(image) pass def preprocess(self, image): imageSize = (INPUT_SIZE, INPUT_SIZE) image = image.convert('L').resize(imageSize, Image.ANTIALIAS) singleArrSize = (INPUT_SIZE, INPUT_SIZE, 1) image = np.asarray(image, dtype='int32') image = image.astype('float32') image /= 255 image = image.reshape(singleArrSize) xShape = (1, INPUT_SIZE, INPUT_SIZE, 1) x = np.zeros(xShape) x[0] = image self.x = x pass def predict(self): x = self.datagen.flow(self.x, batch_size=1, shuffle=False).next() # preImg = Image.fromarray(x.reshape(32, 32) * 255).convert('RGB') # preImg.save("./n.png") prediction = self.model.predict(x) anomalies = np.zeros(1) if self.anomalyModel: anomalies = self.anomalyModel.predict(prediction) predictClass = np.argmax(prediction, axis=1) if anomalies[0]: print("[INFO] predict result : anomaly") return 99 # 其它图标 else: print("[INFO] predict result : " + self.classList[str(predictClass[0])]) return predictClass[0] pass
def main(): host_collector = HostCollector() anomaly_detector = AnomalyDetector() anomaly_collector = AnomalyCollector() result_controller = ResultController() anomaly_writer = AnomalyWriter(whole_result_address) try: # save the begin time begin_time = datetime.datetime.now() detect_time_flag = True global study_ready_time global detect_ready_time with open(url_address, 'rb') as infile: record_num = 0 study_record_num = 0 for line in infile: record = line.strip(' \n').split('\t') if len(record) == 13 and record[_host] != '': a_url = UrlRecord(record) current_model = host_collector.getHostModel(a_url.get_host()) pattern_flag = current_model.getDetectFlag() # Judge Host-model's pattern('Study ready' or 'Study...') if pattern_flag == 'Study ready': if detect_time_flag: detect_time_flag = False study_ready_time = datetime.datetime.now() study_interval = study_ready_time - begin_time study_record_num = record_num print 'Study consuming: %s' % study_interval print 'Study Record: %s' % study_record_num print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \ (current_model.getUrlAmount(), current_model.getDifUrlAmount(), current_model.getSipAmount()) # Detect whether the record is anomaly anomaly_status = anomaly_detector.detect(a_url, current_model) # If the record is detected to be anomaly if anomaly_status['Result'] is True: valid_flag = result_controller.isValid(a_url, anomaly_collector) if valid_flag is True: anomaly_writer.writeResult(a_url, anomaly_status) else: current_model.reStudy() elif pattern_flag == 'Study...': current_model.add_record(a_url) else: raise ValueError('Lancer says: pattern selection error.') record_num += 1 if record_num % 10000 == 0: print 'Record completed: ', record_num, else: detect_record_num = record_num - study_record_num end_time = datetime.datetime.now() detect_ready_time = end_time - study_ready_time print 'Detect Record: %s' % detect_record_num print "Detection consuming: %s" % detect_ready_time print 'Whole system test finished.' # Store the completed Host Model. with open(host_collector_address, 'wb') as save_host_collector: pickle.dump(host_collector, save_host_collector) except IndexError: print 'Lancer says:\n\tIt has error through the system.' raise
class IOTdevice: def __init__(self, device_name, buffer_size=100, upload_to_iotb=False): global plot_this self.device_id = device_name self.anomaly_detector = AnomalyDetector( buffer_size, window_length_sec=10, num_windows=50, plot_time=False, plot=plot_this ) plot_this = False self.upload_to_iotb = upload_to_iotb self.pkts_since_upload = [] self.server_on = False try: r = requests.get(SERVER + "?device_id=d1") self.server_on = True except requests.exceptions.ConnectionError: self.server_on = False # self.window_anomaly_detector = WindowAnomalyDetector(window_length_sec=20, num_windows=30, ML_alg=sklearn.svm.OneClassSVM) def build_model(self, pf): """build database of pktFeatures. Upload every 10 packets. Featurizer is non json so must convert to dict""" get_pkt_type = { ICMP: "ICMP", DNS: "DNS", STP: "STP", DHCP: "DHCP", TCP: "TCP", UDP: "UDP", IP: "IP", ARP: "ARP", Ether: "Ether", } features = {} features["arrival_time"] = pf.arrival_time features["len_bytes"] = pf.len_bytes features["pkt_type"] = get_pkt_type[pf.pkt_type] features["features"] = pf.features self.pkts_since_upload.append(features) if len(self.pkts_since_upload) >= pkts_per_upload: self.upload_model() def upload_model(self): """upload features to SERVER""" payload = {"device_id": self.device_id, "data": self.pkts_since_upload} r = requests.post(SERVER, json=payload) # print payload def download_model(self): """download model if device was identified""" if self.server_on: r = requests.get(SERVER + "?device_id=" + self.device_id) # if model found on server if r.status_code == 200: try: data = pickle.loads(r.content) for each_pkt in data: self.anomaly_detector.add_training_data(Featurizer(each_pkt)) except: # print "Model for device {} not found on server".format(self.device_id) pass def handle_pkt(self, pkt): pf = PktFeaturizer(pkt) if self.server_on: self.build_model(pf) prediction, distance = self.anomaly_detector.predict(pf) self.anomaly_detector.add_training_data(pf) # self.window_anomaly_detector.predict(pf) # self.window_anomaly_detector.add_training_data(pf) return prediction
import pandas as pd from AnomalyDetector import AnomalyDetector from datetime import datetime g = pd.read_csv('../data/gageids.csv', header=None, dtype='str') ad = AnomalyDetector('streamanomaly', datetime.now()) ad.getAnomalies(g[0].values)