Exemplo n.º 1
0
def model_learn(flow_address, host_stored_address):
    host_collector = HostCollector()

    start_time = datetime.datetime.now()
    record_num = 0
    with open(flow_address, 'rb') as infile:
        for line in infile:
            record = line.strip(' \n').split('\t')
            if len(record) == 13 and record[_host] != '':
                current_record = FlowRecord(record)
                current_model = host_collector.get_host_model(current_record['host'])
                current_model.add_record(current_record)
                record_num += 1
                if record_num % 10000 == 0:
                    print 'Record completed: %s' % record_num
    print 'Study Records: %d' % record_num
    print 'Study Host Models: %d' % len(host_collector)
    for host_model in host_collector:
        host_model.generate_feature()
    end_time = datetime.datetime.now()
    print 'Model Learn Consuming: %s' % (end_time - start_time)

    for host_model in host_collector:
        print host_model

    with open(host_stored_address, 'wb') as store_host_collector:
        cpickle.dump(host_collector, store_host_collector)

    print 'LEARN Module Finished.'
Exemplo n.º 2
0
def model_learn(flow_address, host_stored_address):
    host_collector = HostCollector()

    start_time = datetime.datetime.now()
    record_num = 0
    with open(flow_address, 'rb') as infile:
        for line in infile:
            record = line.strip(' \n').split('\t')
            if len(record) == 13 and record[_host] != '':
                current_record = FlowRecord(record)
                current_model = host_collector.get_host_model(
                    current_record['host'])
                current_model.add_record(current_record)
                record_num += 1
                if record_num % 10000 == 0:
                    print 'Record completed: %s' % record_num
    print 'Study Records: %d' % record_num
    print 'Study Host Models: %d' % len(host_collector)
    for host_model in host_collector:
        host_model.generate_feature()
    end_time = datetime.datetime.now()
    print 'Model Learn Consuming: %s' % (end_time - start_time)

    for host_model in host_collector:
        print host_model

    with open(host_stored_address, 'wb') as store_host_collector:
        cpickle.dump(host_collector, store_host_collector)

    print 'LEARN Module Finished.'
Exemplo n.º 3
0
def main():
    host_collector = HostCollector()
    anomaly_detector = AnomalyDetector()
    anomaly_collector = AnomalyCollector()
    result_controller = ResultController()
    anomaly_writer = AnomalyWriter(whole_result_address)

    try:
        # save the begin time
        begin_time = datetime.datetime.now()
        detect_time_flag = True
        global study_ready_time
        global detect_ready_time

        with open(url_address, 'rb') as infile:
            record_num = 0
            study_record_num = 0
            for line in infile:
                record = line.strip(' \n').split('\t')
                if len(record) == 13 and record[_host] != '':
                    a_url = UrlRecord(record)
                    current_model = host_collector.getHostModel(
                        a_url.get_host())
                    pattern_flag = current_model.getDetectFlag()
                    # Judge Host-model's pattern('Study ready' or 'Study...')
                    if pattern_flag == 'Study ready':
                        if detect_time_flag:
                            detect_time_flag = False
                            study_ready_time = datetime.datetime.now()
                            study_interval = study_ready_time - begin_time
                            study_record_num = record_num
                            print 'Study consuming: %s' % study_interval
                            print 'Study Record: %s' % study_record_num
                            print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
                                (current_model.getUrlAmount(), current_model.getDifUrlAmount(),
                                 current_model.getSipAmount())
                        # Detect whether the record is anomaly
                        anomaly_status = anomaly_detector.detect(
                            a_url, current_model)
                        # If the record is detected to be anomaly
                        if anomaly_status['Result'] is True:
                            valid_flag = result_controller.isValid(
                                a_url, anomaly_collector)
                            if valid_flag is True:
                                anomaly_writer.writeResult(
                                    a_url, anomaly_status)
                            else:
                                current_model.reStudy()
                    elif pattern_flag == 'Study...':
                        current_model.add_record(a_url)
                    else:
                        raise ValueError(
                            'Lancer says: pattern selection error.')
                record_num += 1
                if record_num % 10000 == 0:
                    print 'Record completed: ', record_num,
            else:
                detect_record_num = record_num - study_record_num
                end_time = datetime.datetime.now()
                detect_ready_time = end_time - study_ready_time
                print 'Detect Record: %s' % detect_record_num
                print "Detection consuming: %s" % detect_ready_time
                print 'Whole system test finished.'

        # Store the completed Host Model.
        with open(host_collector_address, 'wb') as save_host_collector:
            pickle.dump(host_collector, save_host_collector)

    except IndexError:
        print 'Lancer says:\n\tIt has error through the system.'
        raise
Exemplo n.º 4
0
def main():
    host_collector = HostCollector()
    anomaly_detector = AnomalyDetector()
    anomaly_collector = AnomalyCollector()
    result_controller = ResultController()
    anomaly_writer = AnomalyWriter(whole_result_address)

    try:
        # save the begin time
        begin_time = datetime.datetime.now()
        detect_time_flag = True
        global study_ready_time
        global detect_ready_time

        with open(url_address, 'rb') as infile:
            record_num = 0
            study_record_num = 0
            for line in infile:
                record = line.strip(' \n').split('\t')
                if len(record) == 13 and record[_host] != '':
                    a_url = UrlRecord(record)
                    current_model = host_collector.getHostModel(a_url.get_host())
                    pattern_flag = current_model.getDetectFlag()
                    # Judge Host-model's pattern('Study ready' or 'Study...')
                    if pattern_flag == 'Study ready':
                        if detect_time_flag:
                            detect_time_flag = False
                            study_ready_time = datetime.datetime.now()
                            study_interval = study_ready_time - begin_time
                            study_record_num = record_num
                            print 'Study consuming: %s' % study_interval
                            print 'Study Record: %s' % study_record_num
                            print '\tUrl amount: %s\tDifferent url amount: %s\tDifferent sip: %s\t' % \
                                (current_model.getUrlAmount(), current_model.getDifUrlAmount(),
                                 current_model.getSipAmount())
                        # Detect whether the record is anomaly
                        anomaly_status = anomaly_detector.detect(a_url, current_model)
                        # If the record is detected to be anomaly
                        if anomaly_status['Result'] is True:
                            valid_flag = result_controller.isValid(a_url, anomaly_collector)
                            if valid_flag is True:
                                anomaly_writer.writeResult(a_url, anomaly_status)
                            else:
                                current_model.reStudy()
                    elif pattern_flag == 'Study...':
                        current_model.add_record(a_url)
                    else:
                        raise ValueError('Lancer says: pattern selection error.')
                record_num += 1
                if record_num % 10000 == 0:
                    print 'Record completed: ', record_num,
            else:
                detect_record_num = record_num - study_record_num
                end_time = datetime.datetime.now()
                detect_ready_time = end_time - study_ready_time
                print 'Detect Record: %s' % detect_record_num
                print "Detection consuming: %s" % detect_ready_time
                print 'Whole system test finished.'

        # Store the completed Host Model.
        with open(host_collector_address, 'wb') as save_host_collector:
            pickle.dump(host_collector, save_host_collector)

    except IndexError:
        print 'Lancer says:\n\tIt has error through the system.'
        raise