예제 #1
0
class TestFileSource(object):
    def __init__(self, source_filename, db_file_path, logger):
        filters = [CommentFilter("#"), SpacesFilter()]
        standard_key_transformer = StandardKeyTransformer()
        ip_key_transformer = lambda key, state: "%(category)s.ip" % {"category": state.state_id}
        handlers = [
            StandardConfigSectionHandler(),
            SimpleKeyValueHandler.create_with_known_key_predicate(
                "=", lambda key, state: state.state_id == "services", standard_key_transformer
            ),
            TransformKeyValueHandler.create_with_known_key_predicate(
                "=", lambda key, state: state.state_id == "users", standard_key_transformer, transform_user_fun
            ),
            AggregateKeyValueHandler.create_with_known_key_list(
                "=", ["ip0", "ip1", "ip2", "ip3", "ip4"], ip_key_transformer, lambda old_value, item: old_value + 1, 0
            ),
        ]
        init_state = OrderedDict()
        encoding = "utf-8"
        self._collect_task = FileSourceCollectTask(
            self._source, filters, handlers, init_state, source_filename, encoding, db_file_path, logger
        )

    def collect_stat_data(self):
        self._collect_task.execute()

    _source = "file_source"
def execute():
    source_id = 'ics.conf'
    # source
    ics_conf_source = settings.ICS_CONF_SOURCE
    ics_conf_encoding = settings.ICS_CONF_ENCODING
    # dest
    dest_db_conn_str = settings.DEST_DB_CONN_STR
    # filters and handlers
    filters = filters_def.filters_def
    handlers = handlers_def.handlers_def
    # initial state
    init_state = handlers_def.initial_state_def
    # logger
    logging.config.dictConfig(settings.LOG_CONF)
    root_logger = logging.getLogger('stat_ics_conf_collector.entry_point')
    # file source collect task
    task = FileSourceCollectTask(source_id, filters, handlers, init_state, ics_conf_source, ics_conf_encoding, dest_db_conn_str, root_logger)
    return task.execute()
 def setUp(self):
     self._mox = Mox()
     self._main_logger = self._mox.CreateMock(Logger)
     self._storage_logger = self._mox.CreateMock(Logger)
     source_filename = os.path.abspath("tests/functional_tests/test.conf")
     self._db_manager.__enter__()
     filters = [CommentFilter("#"), SpacesFilter()]
     standard_key_transformer = StandardKeyTransformer()
     ip_key_transformer = lambda key, value, state: "{category:s}.ip".format(category=state.state_id)
     handlers = [
         StandardConfigSectionHandler(),
         SimpleKeyValueHandler.create_with_known_key_predicate(
             "=", lambda key, state: state.state_id == "services", standard_key_transformer
         ),
         TransformKeyListHandler.create_with_known_key_predicate(
             "=", lambda key, state: state.state_id == "users", standard_key_transformer, transform_user_fun
         ),
         AggregateKeyValueHandler.create_with_known_key_list(
             "=", ["ip0", "ip1", "ip2", "ip3", "ip4"], ip_key_transformer, lambda old_value, item: old_value + 1, 0
         ),
         SimpleKeyValueHandler.create_with_known_key_list(
             "=", ["use_local_mail", "use_remote_mail", "use_jabber"], standard_key_transformer
         ),
         SingleKeyHandler.create_with_known_key_list(
             "=", ["use_local_mail", "use_remote_mail", "use_jabber"], standard_key_transformer, "0"
         ),
     ]
     self._collect_task = FileSourceCollectTask(
         "some_source",
         filters,
         handlers,
         OrderedDict(),
         source_filename,
         "utf8",
         self._db_manager.db_filename,
         self._main_logger,
     )
class TestFileSourceCollectTask(TestCase):
    def __init__(self, methodName="runTest"):
        super(TestFileSourceCollectTask, self).__init__(methodName)
        self._mox = None
        self._main_logger = None
        self._storage_logger = None
        self._collect_task = None
        self._db_manager = sqlite_db_manager.SqliteDbManager("../stat_sender_db/create.py")

    def setUp(self):
        self._mox = Mox()
        self._main_logger = self._mox.CreateMock(Logger)
        self._storage_logger = self._mox.CreateMock(Logger)
        source_filename = os.path.abspath("tests/functional_tests/test.conf")
        self._db_manager.__enter__()
        filters = [CommentFilter("#"), SpacesFilter()]
        standard_key_transformer = StandardKeyTransformer()
        ip_key_transformer = lambda key, value, state: "{category:s}.ip".format(category=state.state_id)
        handlers = [
            StandardConfigSectionHandler(),
            SimpleKeyValueHandler.create_with_known_key_predicate(
                "=", lambda key, state: state.state_id == "services", standard_key_transformer
            ),
            TransformKeyListHandler.create_with_known_key_predicate(
                "=", lambda key, state: state.state_id == "users", standard_key_transformer, transform_user_fun
            ),
            AggregateKeyValueHandler.create_with_known_key_list(
                "=", ["ip0", "ip1", "ip2", "ip3", "ip4"], ip_key_transformer, lambda old_value, item: old_value + 1, 0
            ),
            SimpleKeyValueHandler.create_with_known_key_list(
                "=", ["use_local_mail", "use_remote_mail", "use_jabber"], standard_key_transformer
            ),
            SingleKeyHandler.create_with_known_key_list(
                "=", ["use_local_mail", "use_remote_mail", "use_jabber"], standard_key_transformer, "0"
            ),
        ]
        self._collect_task = FileSourceCollectTask(
            "some_source",
            filters,
            handlers,
            OrderedDict(),
            source_filename,
            "utf8",
            self._db_manager.db_filename,
            self._main_logger,
        )

    def tearDown(self):
        self._db_manager.__exit__(None, None, None)

    def test_execute(self):
        data_list = [
            DataItem("gate.ip", 3),
            DataItem("dns.ip", 2),
            DataItem("wins.ip", 1),
            DataItem("users.user", "ivanov,*******"),
            DataItem("users.user", "petrov,***"),
            DataItem("users.user", "sydorov,*********"),
            DataItem("users.user", "kozlov,*********"),
            DataItem("services.http", "80"),
            DataItem("services.ftp", "21"),
            DataItem("options.use_local_mail", "1"),
            DataItem("options.use_remote_mail", "0"),
            DataItem("options.use_jabber", "0"),
        ]
        self._main_logger.getChild("sqlite_storage").AndReturn(self._storage_logger)
        self._main_logger.info("execute() enter")
        self._main_logger.info("_read_file_content() enter")
        self._main_logger.info("_read_file_content() exit")
        self._main_logger.info("_write_data(data_dict) enter")
        self._storage_logger.info("save_data(some_source, data_list) enter")
        for data in data_list:
            self._storage_logger.info("_save_item_impl(some_source, {0!s}) enter".format(data))
            self._storage_logger.info("_save_item_impl(some_source, {0!s}) exit".format(data))
        self._storage_logger.info("save_data(some_source, data_list) exit")
        self._main_logger.info("_write_data(data_dict) exit")
        self._main_logger.info("execute() exit")
        self._mox.ReplayAll()
        now = datetime.now()
        result = self._collect_task.execute()
        self.assertTrue(result)
        actual = self._db_manager.execute_query(
            "select ID, SOURCE, CATEGORY, TIMEMARKER, DATA from STAT_DATA order by ID"
        )
        expected = [
            ("gate.ip", 3),
            ("dns.ip", 2),
            ("wins.ip", 1),
            ("users.user", "ivanov,*******"),
            ("users.user", "petrov,***"),
            ("users.user", "sydorov,*********"),
            ("users.user", "kozlov,*********"),
            ("services.http", "80"),
            ("services.ftp", "21"),
            ("options.use_local_mail", "1"),
            ("options.use_remote_mail", "0"),
            ("options.use_jabber", "0"),
        ]
        self._check_data(now, "some_source", expected, actual)
        self._mox.VerifyAll()

    # spec: datetime, str, [(str, str)], [(int, str, str, str, str)] -> None
    def _check_data(self, now, source_id, expected, actual):
        self.assertEqual(len(expected), len(actual))
        for expected_item in expected:
            actual_items = filter(
                lambda item: item[1] == source_id and item[2] == expected_item[0] and item[4] == expected_item[1],
                actual,
            )
            self.assertEqual(1, len(actual_items))
            actual_item = actual_items[0]
            actual_time = self._str_2_time(actual_item[3])
            self.assertTrue(actual_time - now < timedelta(seconds=10))

    # spec: str -> datetime
    def _str_2_time(self, source_str):
        return datetime.strptime(source_str, "%Y-%m-%d %H:%M:%S")