def _get_process_manager(self):
"""
init all task according to self._commands
return
ProcesserManager
Exception:
ProcesserManagerInitError
"""
process_manager = ProcesserManager()
process_names = []
for task in self._tasks:
process_name = task.get_process_name()
process_names.extend(process_name)
for (module, class_name, args) in process_names:
try:
class_obj = process_manager.init_class(module, class_name)
except ProcesserManagerInitError as e:
logging.warning("Init class_obj [%s] module[%s] failed" % (class_name, module))
raise e
logging.info("Init class_obj [%s] module[%s] successful" % (class_name, module))
ret = class_obj.check_args(args)
if ret != 0:
logging.warning("Check args [%s] for class_obj [%s] module [%s] failed" % \
(str(args), class_name, module))
raise ProcesserManagerInitError("Check args [%s] for class_obj [%s] module [%s] failed" % \
(str(args), class_name, module))
logging.info("Check args [%s] for class_obj [%s] module [%s] successful" % \
(str(args), class_name, module))
return process_manager
class TestTableCheckerTask(unittest.TestCase):
""" test for table checker """
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker")
self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker")
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
#self._p_m.init_class("join_checker", "")
col_check_task = []
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["0", None, None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["int"], [["1", None, None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["int"], [["2", "preprocess_trim.PreprocessTrim", None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["float"], [["3", "preprocess_trim.PreprocessTrim", None]]))
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["4", "preprocess_time.PreprocessTime", None]]))
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["5", "preprocess_time.PreprocessTime", ["%Y:%M:%d"]]]))
line = assemble_parameter("./data/table_checker/file_in", 6, ",", col_check_task)
self._checker = TableCheckerTask(line)
def tearDown(self):
del self._p_m
del self._checker
def test_get_process_name(self):
""" test for get process name """
names = self._checker.get_process_name()
real_names = [
["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \
["col_checker", "col_checker_type.ColTypeChecker", ["int"]], \
["col_checker", "col_checker_type.ColTypeChecker", ["int"]], \
["col_checker", "col_checker_type.ColTypeChecker", ["float"]], \
["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \
["col_checker", "col_checker_time.ColTimeChecker", ["%Y-%m-%d %H:%M:%S"]], \
["preprocesser" , "preprocess_trim.PreprocessTrim", None], \
["preprocesser" , "preprocess_trim.PreprocessTrim", None], \
["preprocesser" , "preprocess_time.PreprocessTime", None], \
["preprocesser" , "preprocess_time.PreprocessTime", ["%Y:%M:%d"]], \
]
for name in names:
self.assertIn(name, real_names)
def test_excute(self):
self._checker.excute(self._p_m)
status_infos = self._checker._status_infos
line_cnt = 10
reals = [8, 9, 9, 9, 9, 10, 10]
for index, s in enumerate(status_infos):
self.assertEqual(s.check_cnt, line_cnt)
self.assertEqual(s.check_fail_cnt, line_cnt - reals[index])
class TestTableJoinCheckerTask(unittest.TestCase):
""" test for table join checker """
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
self._p_m.init_class("join_checker", "join_checker_equal.EqualJoinChecker")
def tearDown(self):
del self._p_m
def test_get_process_name(self):
""" test for get process name"""
join_checker = "join_checker"
args = ["1", "2"]
file1 = "file1"
file2 = "file2"
fields1 = [ \
["1", None, ["C1"]],\
["2", None, None],\
["3", "P3", None],\
["4", "P4", ["C4"]],\
["5", "P5", ["C5.1", "C5.2"]]\
]
fields2 = [["2", None, None]]
line = assemble_parameter(join_checker, args, file1, file2, fields1, fields2)
checker = TableJoinCheckerTask(line)
names = checker.get_process_name()
real_names1 = [["preprocesser", p[1], p[2]] for p in fields1 if p[1] is not None]
real_names2 = [["preprocesser", p[1], p[2]] for p in fields2 if p[1] is not None]
real_names=real_names1
real_names.extend(real_names2)
real_names.append(["join_checker", "join_checker", args])
for name in names:
self.assertIn(name, real_names)
def test_excute(self):
""" test for excute """
join_checker = "join_checker_equal.EqualJoinChecker"
args = None
file1 = "./data/table_join_checker/file1.in"
file2 = "./data/table_join_checker/file2.in"
fields1 = [["0", None, None]]
fields2 = [["0", None, None]]
line = assemble_parameter(join_checker, args, file1, file2, fields1, fields2)
checker = TableJoinCheckerTask(line)
checker.excute(self._p_m)
status_info = checker._status_infos[0]
self.assertEqual(status_info.check_cnt, 7)
self.assertEqual(status_info.check_fail_cnt, 3)
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker")
self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker")
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker")
self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker")
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
#self._p_m.init_class("join_checker", "")
col_check_task = []
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["0", None, None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["int"], [["1", None, None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["int"], [["2", "preprocess_trim.PreprocessTrim", None]]))
col_check_task.append(assemble_sub_task("col_checker_type.ColTypeChecker", \
["float"], [["3", "preprocess_trim.PreprocessTrim", None]]))
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["4", "preprocess_time.PreprocessTime", None]]))
col_check_task.append(assemble_sub_task("col_checker_time.ColTimeChecker", \
["%Y-%m-%d %H:%M:%S"], [["5", "preprocess_time.PreprocessTime", ["%Y:%M:%d"]]]))
line = assemble_parameter("./data/table_checker/file_in", 6, ",", col_check_task)
self._checker = TableCheckerTask(line)
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
self._p_m.init_class("join_checker", "join_checker_equal.EqualJoinChecker")
class TestTableCheckerSubTask(unittest.TestCase):
""" test for table checker """
def setUp(self):
""" init a great process manager"""
self._p_m = ProcesserManager()
self._p_m.init_class("col_checker", "col_checker_time.ColTimeChecker")
self._p_m.init_class("col_checker", "col_checker_type.ColTypeChecker")
self._p_m.init_class("preprocesser", "preprocess_time.PreprocessTime")
self._p_m.init_class("preprocesser", "preprocess_trim.PreprocessTrim")
#self._p_m.init_class("join_checker", "")
def tearDown(self):
del self._p_m
def test_get_process_name(self):
""" test for get process name """
task_name = "test1"
checker = "test1_checker"
args = ["1", "2"]
field_args = [ \
["1", None, ["C1"]],\
["2", None, None],\
["3", "P3", None],\
["4", "P4", ["C4"]],\
["5", "P5", ["C5.1", "C5.2"]]\
]
fields = FieldList(assemble_field_list(field_args))
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
names = table_checker.get_process_name()
real_names = [["preprocesser", p[1], p[2]] for p in field_args if p[1] is not None]
real_names.append(["col_checker", checker, args])
# check process name
for name in names:
self.assertIn(name, real_names)
def test_excute_type_checker(self):
"""test for excute"""
task_name = "test1"
checker = "col_checker_type.ColTypeChecker"
args = [u"int"]
field_args = [["0", None, None]]
fields = FieldList(assemble_field_list(field_args))
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
# 1. check for empty pm
p_manager = ProcesserManager()
self.assertRaises(ProcesserManagerLocateError, table_checker.excute, p_manager, [])
cols = ["13"]
self.assertTrue(table_checker.excute(self._p_m, cols))
cols = ["dsj13"]
self.assertFalse(table_checker.excute(self._p_m, cols))
field_args = [["0", "preprocess_trim.PreprocessTrim", None]]
fields = FieldList(assemble_field_list(field_args))
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
cols = ["13"]
self.assertTrue(table_checker.excute(self._p_m, cols))
cols = [' 13 "']
self.assertTrue(table_checker.excute(self._p_m, cols))
cols = ['"13df"']
self.assertFalse(table_checker.excute(self._p_m, cols))
args = [u"int", u"float"]
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
cols = ['"13.3"']
self.assertTrue(table_checker.excute(self._p_m, cols))
def test_excute_time_checker(self):
""" test for excute time checker """
task_name = "test2"
checker = "col_checker_time.ColTimeChecker"
args = [u"%Y%m%d"]
field_args = [["0", None, None]]
fields = FieldList(assemble_field_list(field_args))
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
cols = ["20150302"]
self.assertTrue(table_checker.excute(self._p_m, cols))
cols = ["dsj13"]
self.assertFalse(table_checker.excute(self._p_m, cols))
args = [u"%Y-%m-%d %H:%M:%S"]
field_args = [["0", "preprocess_time.PreprocessTime", [u"%Y-%m-%d"]]]
fields = FieldList(assemble_field_list(field_args))
table_checker = TableCheckerSubTask(task_name, checker, args, fields)
cols = ["2015-03-02"]
self.assertTrue(table_checker.excute(self._p_m, cols))
cols = ["20150302"]
self.assertFalse(table_checker.excute(self._p_m, cols))
