class TestTableCheckerTask(unittest.TestCase): """ test for table checker """ def setUp(self): """ init a great process manager""" self._p_m = ProcesserManager() self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker") self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker") self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime") self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim") #self._p_m.init_class("join_checker", "") col_check_task = [] col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \ ["%Y-%m-%d %H:%M:%S"], [["0", None, None]])) col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \ ["int"], [["1", None, None]])) col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \ ["int"], [["2", "preprocess_trim.PreprocessTrim", None]])) col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \ ["float"], [["3", "preprocess_trim.PreprocessTrim", None]])) col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \ ["%Y-%m-%d %H:%M:%S"], [["4", "preprocess_time.PreprocessTime", None]])) col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \ ["%Y-%m-%d %H:%M:%S"], [["5", "preprocess_time.PreprocessTime", ["%Y:%M:%d"]]])) line = assemble_parameter("./data/table_checker/file_in", 6, ",", col_check_task) self._checker = TableCheckerTask(line) def tearDown(self): del self._p_m del self._checker def test_get_process_name(self): """ test for get process name """ names = self._checker.get_process_name() real_names = [ ["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \ ["col_checker", "col_checker_type.ColTypeChecker", ["int"]], \ ["col_checker", "col_checker_type.ColTypeChecker", ["int"]], \ ["col_checker", "col_checker_type.ColTypeChecker", ["float"]], \ ["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \ ["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \ ["preprocesser" , "preprocess_trim.PreprocessTrim", None], \ ["preprocesser" , "preprocess_trim.PreprocessTrim", None], \ ["preprocesser" , "preprocess_time.PreprocessTime", None], \ ["preprocesser" , "preprocess_time.PreprocessTime", ["%Y:%M:%d"]], \ ] for name in names: self.assertIn(name, real_names) def test_excute(self): self._checker.excute(self._p_m) status_infos = self._checker._status_infos line_cnt = 10 reals = [8, 9, 9, 9, 9, 10, 10] for index, s in enumerate(status_infos): self.assertEqual(s.check_cnt, line_cnt) self.assertEqual(s.check_fail_cnt, line_cnt - reals[index])
class TestTableJoinCheckerTask(unittest.TestCase): """ test for table join checker """ def setUp(self): """ init a great process manager""" self._p_m = ProcesserManager() self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime") self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim") self._p_m.init_class("join_checker", "join_checker_equal.EqualJoinChecker") def tearDown(self): del self._p_m def test_get_process_name(self): """ test for get process name""" join_checker = "join_checker" args = ["1", "2"] file1 = "file1" file2 = "file2" fields1 = [ \ ["1", None, ["C1"]],\ ["2", None, None],\ ["3", "P3", None],\ ["4", "P4", ["C4"]],\ ["5", "P5", ["C5.1", "C5.2"]]\ ] fields2 = [["2", None, None]] line = assemble_parameter(join_checker, args, file1, file2, fields1, fields2) checker = TableJoinCheckerTask(line) names = checker.get_process_name() real_names1 = [["preprocesser", p[1], p[2]] for p in fields1 if p[1] is not None] real_names2 = [["preprocesser", p[1], p[2]] for p in fields2 if p[1] is not None] real_names=real_names1 real_names.extend(real_names2) real_names.append(["join_checker", "join_checker", args]) for name in names: self.assertIn(name, real_names) def test_excute(self): """ test for excute """ join_checker = "join_checker_equal.EqualJoinChecker" args = None file1 = "./data/table_join_checker/file1.in" file2 = "./data/table_join_checker/file2.in" fields1 = [["0", None, None]] fields2 = [["0", None, None]] line = assemble_parameter(join_checker, args, file1, file2, fields1, fields2) checker = TableJoinCheckerTask(line) checker.excute(self._p_m) status_info = checker._status_infos[0] self.assertEqual(status_info.check_cnt, 7) self.assertEqual(status_info.check_fail_cnt, 3)
def _get_process_manager(self): """ init all task according to self._commands return ProcesserManager Exception: ProcesserManagerInitError """ process_manager = ProcesserManager() process_names = [] for task in self._tasks: process_name = task.get_process_name() process_names.extend(process_name) for (module, class_name, args) in process_names: try: class_obj = process_manager.init_class(module, class_name) except ProcesserManagerInitError as e: logging.warning("Init class_obj [%s] module[%s] failed" % (class_name, module)) raise e logging.info("Init class_obj [%s] module[%s] successful" % (class_name, module)) ret = class_obj.check_args(args) if ret != 0: logging.warning("Check args [%s] for class_obj [%s] module [%s] failed" % \ (str(args), class_name, module)) raise ProcesserManagerInitError("Check args [%s] for class_obj [%s] module [%s] failed" % \ (str(args), class_name, module)) logging.info("Check args [%s] for class_obj [%s] module [%s] successful" % \ (str(args), class_name, module)) return process_manager
class TestTableCheckerSubTask(unittest.TestCase): """ test for table checker """ def setUp(self): """ init a great process manager""" self._p_m = ProcesserManager() self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker") self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker") self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime") self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim") #self._p_m.init_class("join_checker", "") def tearDown(self): del self._p_m def test_get_process_name(self): """ test for get process name """ task_name = "test1" checker = "test1_checker" args = ["1", "2"] field_args = [ \ ["1", None, ["C1"]],\ ["2", None, None],\ ["3", "P3", None],\ ["4", "P4", ["C4"]],\ ["5", "P5", ["C5.1", "C5.2"]]\ ] fields = FieldList(assemble_field_list(field_args)) table_checker = TableCheckerSubTask(task_name, checker, args, fields) names = table_checker.get_process_name() real_names = [["preprocesser", p[1], p[2]] for p in field_args if p[1] is not None] real_names.append(["col_checker", checker, args]) # check process name for name in names: self.assertIn(name, real_names) def test_excute_type_checker(self): """test for excute""" task_name = "test1" checker = "col_checker_type.ColTypeChecker" args = [u"int"] field_args = [["0", None, None]] fields = FieldList(assemble_field_list(field_args)) table_checker = TableCheckerSubTask(task_name, checker, args, fields) # 1. check for empty pm p_manager = ProcesserManager() self.assertRaises(ProcesserManagerLocateError, table_checker.excute, p_manager, []) cols = ["13"] self.assertTrue(table_checker.excute(self._p_m, cols)) cols = ["dsj13"] self.assertFalse(table_checker.excute(self._p_m, cols)) field_args = [["0", "preprocess_trim.PreprocessTrim", None]] fields = FieldList(assemble_field_list(field_args)) table_checker = TableCheckerSubTask(task_name, checker, args, fields) cols = ["13"] self.assertTrue(table_checker.excute(self._p_m, cols)) cols = [' 13 "'] self.assertTrue(table_checker.excute(self._p_m, cols)) cols = ['"13df"'] self.assertFalse(table_checker.excute(self._p_m, cols)) args = [u"int", u"float"] table_checker = TableCheckerSubTask(task_name, checker, args, fields) cols = ['"13.3"'] self.assertTrue(table_checker.excute(self._p_m, cols)) def test_excute_time_checker(self): """ test for excute time checker """ task_name = "test2" checker = "col_checker_time.ColTimeChecker" args = [u"%Y%m%d"] field_args = [["0", None, None]] fields = FieldList(assemble_field_list(field_args)) table_checker = TableCheckerSubTask(task_name, checker, args, fields) cols = ["20150302"] self.assertTrue(table_checker.excute(self._p_m, cols)) cols = ["dsj13"] self.assertFalse(table_checker.excute(self._p_m, cols)) args = [u"%Y-%m-%d %H:%M:%S"] field_args = [["0", "preprocess_time.PreprocessTime", [u"%Y-%m-%d"]]] fields = FieldList(assemble_field_list(field_args)) table_checker = TableCheckerSubTask(task_name, checker, args, fields) cols = ["2015-03-02"] self.assertTrue(table_checker.excute(self._p_m, cols)) cols = ["20150302"] self.assertFalse(table_checker.excute(self._p_m, cols))