def model_learn(flow_address, host_stored_address):
host_collector = HostCollector()
start_time = datetime.datetime.now()
record_num = 0
with open(flow_address, 'rb') as infile:
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
current_record = FlowRecord(record)
current_model = host_collector.get_host_model(current_record['host'])
current_model.add_record(current_record)
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: %s' % record_num
print 'Study Records: %d' % record_num
print 'Study Host Models: %d' % len(host_collector)
for host_model in host_collector:
host_model.generate_feature()
end_time = datetime.datetime.now()
print 'Model Learn Consuming: %s' % (end_time - start_time)
for host_model in host_collector:
print host_model
with open(host_stored_address, 'wb') as store_host_collector:
cpickle.dump(host_collector, store_host_collector)
print 'LEARN Module Finished.'
def model_learn(flow_address, host_stored_address):
host_collector = HostCollector()
start_time = datetime.datetime.now()
record_num = 0
with open(flow_address, 'rb') as infile:
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
current_record = FlowRecord(record)
current_model = host_collector.get_host_model(
current_record['host'])
current_model.add_record(current_record)
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: %s' % record_num
print 'Study Records: %d' % record_num
print 'Study Host Models: %d' % len(host_collector)
for host_model in host_collector:
host_model.generate_feature()
end_time = datetime.datetime.now()
print 'Model Learn Consuming: %s' % (end_time - start_time)
for host_model in host_collector:
print host_model
with open(host_stored_address, 'wb') as store_host_collector:
cpickle.dump(host_collector, store_host_collector)
print 'LEARN Module Finished.'
def main():
host_collector = HostCollector()
anomaly_detector = AnomalyDetector()
anomaly_collector = AnomalyCollector()
result_controller = ResultController()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
study_record_num = 0
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
a_url = UrlRecord(record)
current_model = host_collector.getHostModel(
a_url.get_host())
pattern_flag = current_model.getDetectFlag()
# Judge Host-model's pattern('Study ready' or 'Study...')
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
study_record_num = record_num
print 'Study consuming: %s' % study_interval
print 'Study Record: %s' % study_record_num
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
(current_model.getUrlAmount(), current_model.getDifUrlAmount(),
current_model.getSipAmount())
# Detect whether the record is anomaly
anomaly_status = anomaly_detector.detect(
a_url, current_model)
# If the record is detected to be anomaly
if anomaly_status['Result'] is True:
valid_flag = result_controller.isValid(
a_url, anomaly_collector)
if valid_flag is True:
anomaly_writer.writeResult(
a_url, anomaly_status)
else:
current_model.reStudy()
elif pattern_flag == 'Study...':
current_model.add_record(a_url)
else:
raise ValueError(
'Lancer says: pattern selection error.')
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: ', record_num,
else:
detect_record_num = record_num - study_record_num
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print 'Detect Record: %s' % detect_record_num
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
# Store the completed Host Model.
with open(host_collector_address, 'wb') as save_host_collector:
pickle.dump(host_collector, save_host_collector)
except IndexError:
print 'Lancer says:\n\tIt has error through the system.'
raise
def main():
host_collector = HostCollector()
anomaly_detector = AnomalyDetector()
anomaly_collector = AnomalyCollector()
result_controller = ResultController()
anomaly_writer = AnomalyWriter(whole_result_address)
try:
# save the begin time
begin_time = datetime.datetime.now()
detect_time_flag = True
global study_ready_time
global detect_ready_time
with open(url_address, 'rb') as infile:
record_num = 0
study_record_num = 0
for line in infile:
record = line.strip(' \n').split('\t')
if len(record) == 13 and record[_host] != '':
a_url = UrlRecord(record)
current_model = host_collector.getHostModel(a_url.get_host())
pattern_flag = current_model.getDetectFlag()
# Judge Host-model's pattern('Study ready' or 'Study...')
if pattern_flag == 'Study ready':
if detect_time_flag:
detect_time_flag = False
study_ready_time = datetime.datetime.now()
study_interval = study_ready_time - begin_time
study_record_num = record_num
print 'Study consuming: %s' % study_interval
print 'Study Record: %s' % study_record_num
print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
(current_model.getUrlAmount(), current_model.getDifUrlAmount(),
current_model.getSipAmount())
# Detect whether the record is anomaly
anomaly_status = anomaly_detector.detect(a_url, current_model)
# If the record is detected to be anomaly
if anomaly_status['Result'] is True:
valid_flag = result_controller.isValid(a_url, anomaly_collector)
if valid_flag is True:
anomaly_writer.writeResult(a_url, anomaly_status)
else:
current_model.reStudy()
elif pattern_flag == 'Study...':
current_model.add_record(a_url)
else:
raise ValueError('Lancer says: pattern selection error.')
record_num += 1
if record_num % 10000 == 0:
print 'Record completed: ', record_num,
else:
detect_record_num = record_num - study_record_num
end_time = datetime.datetime.now()
detect_ready_time = end_time - study_ready_time
print 'Detect Record: %s' % detect_record_num
print "Detection consuming: %s" % detect_ready_time
print 'Whole system test finished.'
# Store the completed Host Model.
with open(host_collector_address, 'wb') as save_host_collector:
pickle.dump(host_collector, save_host_collector)
except IndexError:
print 'Lancer says:\n\tIt has error through the system.'
raise