def test_execute_ok3(self): try: os.makedirs(self._data_dir, exist_ok=True) # create test file csv_list1 = [["key", "data"], ["c1", "spam"], ["c2", "spam"]] with open(os.path.join(self._data_dir, "test1.csv"), "w") as t1: writer = csv.writer(t1) writer.writerows(csv_list1) # set the essential attributes instance = CsvConcat() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test.*\.csv") Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", "test.csv") instance.execute() with open(os.path.join(self._data_dir, "test.csv")) as t: reader = csv.reader(t) concatenated_list = [row for row in reader] finally: shutil.rmtree(self._data_dir) assert concatenated_list == [ ["key", "data"], ["c1", "spam"], ["c2", "spam"], ]
def test_end_with_noerror(self): if sys.version_info.minor < 6: # ignore test if python version is less 3.6(assert_called is not supported) return with ExitStack() as stack: mock_before_step = stack.enter_context( patch("cliboa.core.listener.StepStatusListener.before_step")) mock_error_step = stack.enter_context( patch("cliboa.core.listener.StepStatusListener.error_step")) mock_after_step = stack.enter_context( patch("cliboa.core.listener.StepStatusListener.after_step")) mock_post_step = stack.enter_context( patch( "cliboa.core.listener.StepStatusListener.after_completion") ) step = SampleCustomStep() Helper.set_property(step, "logger", LisboaLog.get_logger(step.__class__.__name__)) Helper.set_property(step, "listeners", [StepStatusListener()]) executor = SingleProcExecutor([step]) executor.execute_steps(None) mock_before_step.assert_called_once() mock_error_step.assert_not_called() mock_after_step.assert_called_once() mock_post_step.assert_called_once()
def test_source_path_reader_with_none(self): instance = SampleCustomStep() Helper.set_property(instance, "logger", LisboaLog.get_logger(instance.__class__.__name__)) ret = instance._source_path_reader(None) assert ret is None
def test_multi_process_error_stop(self): py_info = sys.version_info major_ver = py_info[0] minor_ver = py_info[1] py_ver = int(str(major_ver) + str(minor_ver)) log = LisboaLog.get_logger(self.__class__.__name__) log.info(minor_ver) if py_ver >= self.MULTI_PROC_SUPPORT_VER: step1 = SampleStep() Helper.set_property(step1, "logger", LisboaLog.get_logger(step1.__class__.__name__)) step2 = ErrorSampleStep() Helper.set_property(step2, "logger", LisboaLog.get_logger(step2.__class__.__name__)) q = StepQueue() q.force_continue = False setattr(ScenarioQueue, "step_queue", q) executor = MultiProcExecutor([step1, step2]) try: executor.execute_steps(None) self.fail("Error must be occured") except StepExecutionFailed: pass
def test_execute_ok_with_remain_column_numbers(self): # create test csv os.makedirs(self._data_dir, exist_ok=True) test_csv = os.path.join(self._data_dir, "test.csv") test_csv_data = [["1", "spam"], ["2", "spam"]] with open(test_csv, "w") as t: writer = csv.writer(t) writer.writerows(test_csv_data) t.flush() # set the essential attributes instance = CsvColumnExtract() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", "test.csv") Helper.set_property(instance, "dest_dir", self._data_dir) remain_column_number = 1 Helper.set_property(instance, "column_numbers", remain_column_number) try: instance.execute() output_file = os.path.join(self._data_dir, "test.csv") with open(output_file, "r") as o: reader = csv.DictReader(o) for r in reader: assert r[test_csv_data[0][0]] == test_csv_data[1][0] finally: shutil.rmtree(self._data_dir)
def __create_di_instance(self, cls_attrs): """ Create an instance to be injected Args: step class attributes Retrurn: DI attribute names, DI instances """ di_keys = [] di_instance = None di_instances = [] di_params = None for k in cls_attrs.keys(): if k in self.DI_KEYS: di_keys.append(k) di_params = cls_attrs.get(k) valid = DIScenarioFormat(k, di_params) valid() di_cls = di_params["class"] di_cls = globals()[di_cls] di_instance = di_cls() if di_instance: self._logger.debug( "An instance %s to be injected exists." % di_instance ) del di_params["class"] # set attributes to instance if di_params: for k, v in di_params.items(): Helper.set_property(di_instance, k, v) di_instances.append(di_instance) return di_keys, di_instances
def test_source_path_reader_with_content(self): instance = SampleCustomStep() Helper.set_property(instance, "logger", LisboaLog.get_logger(instance.__class__.__name__)) ret = instance._source_path_reader({"content": "test"}) with open(ret, "r") as fp: actual = fp.read() assert "test" == actual
def __create_instance(self, s_dict, yaml_scenario_list): cls_name = s_dict["class"] self._logger.debug("Create %s instance" % cls_name) if self.__is_custom_cls(cls_name) is True: from cliboa.core.factory import CustomInstanceFactory instance = CustomInstanceFactory.create(cls_name) else: cls = globals()[cls_name] instance = cls() cls_attrs_dict = {} if isinstance(yaml_scenario_list, list) and "arguments" in s_dict.keys(): cls_attrs_dict = s_dict["arguments"] # loop and set class attribute di_key = None di_instance = None values = {} if cls_attrs_dict: cls_attrs_dict = self.__extract_with_vars(cls_attrs_dict) # check if the keys of dependency injection di_keys, di_instances = self.__create_di_instance(cls_attrs_dict) if di_keys and di_instances: di_keys_and_instances = zip(di_keys, di_instances) for di_key, di_instance in di_keys_and_instances: self._logger.debug( "Inject %s to %s object." % (di_instance, instance) ) # Injection setattr(instance, di_key, di_instance) del cls_attrs_dict[di_key] pattern = re.compile(r"{{(.*?)}}") for yaml_k, yaml_v in cls_attrs_dict.items(): # if value includes {{ var }}, replace value specified by with_vars if isinstance(yaml_v, str): matches = pattern.findall(yaml_v) for match in matches: var_name = match.strip() yaml_v = self.__replace_vars(yaml_v, var_name) Helper.set_property(instance, yaml_k, yaml_v) values[yaml_k] = yaml_v base_args = ["step", "symbol", "parallel", "io", "listeners"] for arg in base_args: if arg == "listeners": self._append_listeners(instance, s_dict.get(arg), values) else: Helper.set_property(instance, arg, s_dict.get(arg)) return instance
def _append_listeners(self, instance, args): listeners = [StepStatusListener()] if args is not None: from cliboa.core.factory import CustomInstanceFactory if type(args) is str: listeners.append(CustomInstanceFactory.create(args)) elif type(args) is list: for arg in args: listeners.append(CustomInstanceFactory.create(arg)) Helper.set_property(instance, "listeners", listeners)
def test_excute_ng_input_files(self): with pytest.raises(InvalidCount) as execinfo: # set the essential attributes instance = CsvConcat() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_filenames", ["test1.csv"]) Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", "test.csv") instance.execute() assert "Two or more input files are required." in str(execinfo.value)
def test_execute_ng_invalid_dbname(self): """ sqlite db does not exist. """ try: instance = BaseSqlite() db_file = os.path.join(self._db_dir, "spam.db") Helper.set_property(instance, "dbname", db_file) Helper.set_property(instance, "tblname", "spam_table") instance.execute() except Exception as e: tb = sys.exc_info()[2] assert "not found" in "{0}".format(e.with_traceback(tb))
def test_convert_header(self): try: # create test file csv_list = [["key", "data"], ["1", "spam"], ["2", "spam"], ["3", "spam"]] os.makedirs(self._data_dir, exist_ok=True) test_csv = os.path.join(self._data_dir, "test.csv") with open(test_csv, "w") as t: writer = csv.writer(t) writer.writerows(csv_list) # set the essential attributes instance = CsvConvert() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test\.csv") Helper.set_property(instance, "headers", [{ "key": "new_key" }, { "data": "new_data" }]) instance.execute() with open(test_csv, "r") as t: reader = csv.reader(t) line = next(reader) finally: shutil.rmtree(self._data_dir) assert line == ["new_key", "new_data"]
def test_ok(self): if sys.version_info.minor < 6: # ignore test if python version is less 3.6(assert_called is not supported) return step = SampleCustomStep() Helper.set_property(step, "logger", LisboaLog.get_logger(step.__class__.__name__)) clz = CustomStepListener() values = {"test_key": "test_value"} clz.__dict__.update(values) Helper.set_property(step, "listeners", [clz]) executor = SingleProcExecutor([step]) executor.execute_steps(None)
def _create_instance(self): instance = SqliteExport() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "dbname", self._DB_NAME) Helper.set_property(instance, "dest_path", self._RESULT_FILE) Helper.set_property(instance, "tblname", self._TBL_NAME) return instance
def __invoke_steps(self, yaml_scenario_list): """ Create executable instance and push them to queue Args: yaml_scenario_list: parsed yaml list """ self._logger.info("Start to invoke scenario") # Create queue to save step instances q = StepQueue() for s_dict in yaml_scenario_list: if "multi_process_count" in s_dict.keys(): q.multi_proc_cnt = s_dict.get("multi_process_count") continue if "force_continue" in s_dict.keys(): q.force_continue = s_dict.get("force_continue") continue instances = [] if "parallel" in s_dict.keys(): for row in s_dict.get("parallel"): instance = self.__create_instance(row, yaml_scenario_list) Helper.set_property( instance, "logger", LisboaLog.get_logger(instance.__class__.__name__), ) instances.append(instance) StepArgument._put(row["step"], instance) else: instance = self.__create_instance(s_dict, yaml_scenario_list) Helper.set_property( instance, "logger", LisboaLog.get_logger(instance.__class__.__name__), ) instances.append(instance) StepArgument._put(s_dict["step"], instance) # Put instance to queue q.push(instances) # save queue to static area setattr(ScenarioQueue, "step_queue", q) self._logger.info("Finish to invoke scenario")
def _append_listeners(self, instance, args, values): listeners = [StepStatusListener()] if args is not None: from cliboa.core.factory import CustomInstanceFactory if type(args) is str: clz = CustomInstanceFactory.create(args) clz.__dict__.update(values) listeners.append(clz) elif type(args) is list: for arg in args: clz = CustomInstanceFactory.create(arg) clz.__dict__.update(values) listeners.append(CustomInstanceFactory.create(clz)) Helper.set_property(instance, "listeners", listeners)
def test_execute_ok(self): try: os.makedirs(self._data_dir) instance = HttpDownload() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) # use Postman echo Helper.set_property(instance, "src_url", "https://postman-echo.com") Helper.set_property(instance, "src_pattern", "get?foo1=bar1&foo2=bar2") Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", "test.result") instance.execute() f = open(os.path.join(self._data_dir, "test.result"), "r") result = f.read() f.close() finally: shutil.rmtree(self._data_dir) assert "postman-echo.com" in result
def test_logging_mask_password(self): """ In log file, 'password' is masked. """ instance = SampleCustomStep() instance._logger = LisboaLog.get_logger(__name__) Helper.set_property(instance, "logger", LisboaLog.get_logger(instance.__class__.__name__)) Helper.set_property(instance, "password", "test") instance.trigger() ret = False with open(self._log_file, mode="r", encoding="utf-8") as f: for line in f: if "password : ****" in line: ret = True break self.assertTrue(ret)
def test_ok_5(self): """ Input csv file is only one. Sqlite database already exists. Csv columns and table columns are not the same. force_insert is True. """ TEST_FILE_1 = "sqlite_write_test_1.csv" TEST_FILE_2 = "sqlite_write_test_2.csv" try: with open(TEST_FILE_1, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f) writer.writerow(["No", "TEXT"]) writer.writerow(["1", "A"]) with open(TEST_FILE_2, "w", encoding="utf-8", newline="") as f: writer = csv.writer(f) writer.writerow(["No", "NAME"]) writer.writerow(["2", "John"]) instance = self._create_instance(TEST_FILE_1, True) instance.execute() instance = self._create_instance(TEST_FILE_2, False) Helper.set_property(instance, "force_insert", True) instance.execute() adapter = SqliteAdapter() adapter.connect(self.DB_NAME) cur = adapter.fetch( "SELECT * FROM %s" % self.TBL_NAME, row_factory=self._dict_factory ) count = 0 for row in cur: if count == 0: assert row == {"No": "1", "TEXT": "A", "NAME": None} elif count == 1: assert row == {"No": "2", "TEXT": None, "NAME": "John"} count += 1 finally: self._clean(self.DB_NAME) self._clean(TEST_FILE_1) self._clean(TEST_FILE_2)
def test_source_path_reader_with_path(self): try: os.makedirs(self._data_dir) dummy_pass = os.path.join(self._data_dir, "id_rsa") with open(dummy_pass, "w") as f: f.write("test") instance = SampleCustomStep() Helper.set_property( instance, "logger", LisboaLog.get_logger(instance.__class__.__name__)) ret = instance._source_path_reader({"file": dummy_pass}) assert ret == dummy_pass with open(ret, "r") as fp: actual = fp.read() assert "test" == actual finally: shutil.rmtree(self._data_dir)
def test_ok_3(self): """ Insert plural records(some records are the same) to a table and export the table with no_duplicate option. """ test_data = [ { "No": 1, "TEXT": "AAA" }, { "No": 2, "TEXT": "BBB" }, { "No": 2, "TEXT": "BBB" }, ] try: self._insert_test_data(test_data) instance = self._create_instance() Helper.set_property(instance, "order", ["No"]) Helper.set_property(instance, "no_duplicate", True) instance.execute() with open(self._RESULT_FILE, "r") as o: reader = csv.DictReader(o) for i, row in enumerate(reader): if i == 0: assert "1" == row.get("No") assert "AAA" == row.get("TEXT") elif i == 1: assert "2" == row.get("No") assert "BBB" == row.get("TEXT") else: pytest.fail("Must not be reached") finally: self._clean(self._DB_NAME) self._clean(self._RESULT_FILE)
def test_multi_process_error_continue(self): py_info = sys.version_info major_ver = py_info[0] minor_ver = py_info[1] py_ver = int(str(major_ver) + str(minor_ver)) if py_ver >= self.MULTI_PROC_SUPPORT_VER: step1 = SampleStep() Helper.set_property(step1, "logger", LisboaLog.get_logger(step1.__class__.__name__)) step2 = ErrorSampleStep() Helper.set_property(step2, "logger", LisboaLog.get_logger(step2.__class__.__name__)) q = StepQueue() q.force_continue = True setattr(ScenarioQueue, "step_queue", q) executor = MultiProcExecutor([step1, step2]) executor.execute_steps(None)
def test_sort(self): try: # create test file csv_list = [["key", "data"], ["1", "A"], ["3", "C"], ["2", "B"]] result_dir = os.path.join(self._data_dir, "result") os.makedirs(result_dir, exist_ok=True) test_csv1 = os.path.join(self._data_dir, "test1.csv") with open(test_csv1, "w") as t: writer = csv.writer(t) writer.writerows(csv_list) test_csv2 = os.path.join(self._data_dir, "test2.csv") with open(test_csv2, "w") as t: writer = csv.writer(t) writer.writerows(csv_list) # set the essential attributes instance = CsvSort() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test.*\.csv") Helper.set_property(instance, "dest_dir", result_dir) Helper.set_property(instance, "order", ["key"]) instance.execute() files = glob(os.path.join(result_dir, "test*.csv")) assert 2 == len(files) for file in files: with open(file, mode='r', encoding="utf-8") as f: reader = csv.DictReader(f) for i, row in enumerate(reader): if i == 0: assert "1" == row.get("key") elif i == 1: assert "2" == row.get("key") elif i == 2: assert "3" == row.get("key") finally: shutil.rmtree(self._data_dir)
def test_execute_ok(self): try: # create test file os.makedirs(self._data_dir, exist_ok=True) excel_file = os.path.join(self._data_dir, "test.xlxs") workbook = xlsxwriter.Workbook(excel_file) workbook.close() # set the essential attributes instance = ExcelConvert() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test\.xlxs") Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", "test.csv") instance.execute() exists_csv = glob(os.path.join(self._data_dir, "test.csv")) finally: shutil.rmtree(self._data_dir) assert "test.csv" in exists_csv[0]
def test_excute_ng_multiple_src(self): with pytest.raises(InvalidCount) as execinfo: try: # create test file os.makedirs(self._data_dir, exist_ok=True) excel_file = os.path.join(self._data_dir, "test1.xlxs") open(excel_file, "w").close() excel_file2 = os.path.join(self._data_dir, "test2.xlxs") open(excel_file2, "w").close() # set the essential attributes instance = ExcelConvert() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test(.*)\.xlxs") Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", r"test(.*).xlxs") instance.execute() finally: shutil.rmtree(self._data_dir) assert "must be only one" in str(execinfo.value)
def test_execute_ng_invalid_extension(self): with pytest.raises(InvalidFormat) as execinfo: try: # create test file os.makedirs(self._data_dir, exist_ok=True) excel_file = os.path.join(self._data_dir, "test.xlxs") open(excel_file, "w").close() excel_file2 = os.path.join(self._data_dir, "test.xlxs.bk") open(excel_file2, "w").close() # set the essential attributes instance = ExcelConvert() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", "test.xlxs") Helper.set_property(instance, "dest_dir", self._data_dir) Helper.set_property(instance, "dest_pattern", "test.xlxs") instance.execute() finally: shutil.rmtree(self._data_dir) assert "not supported" in str(execinfo.value)
def test_compress_zip(self): try: # create test file os.makedirs(self._data_dir, exist_ok=True) test_file = os.path.join(self._data_dir, "test.txt") with open(test_file, "w") as t: t.write("ABCDEF") # set the essential attributes instance = FileArchive() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "src_dir", self._data_dir) Helper.set_property(instance, "src_pattern", r"test\.txt") Helper.set_property(instance, "format", "zip") Helper.set_property(instance, "dest_pattern", "foo") instance.execute() files = glob(os.path.join(self._data_dir, "*.zip")) assert 1 == len(files) assert "foo.zip" == os.path.basename(files[0]) finally: shutil.rmtree(self._data_dir)
def test_execute_ok(self, m_get_client): with tempfile.TemporaryDirectory() as tmp_dir: m_get_object = m_get_client.return_value.get_object m_pagenate = m_get_client.return_value.get_paginator.return_value.paginate m_contents = [{"Contents": [{"Key": "spam"}]}] m_pagenate.return_value = m_contents instance = S3Download() Helper.set_property(instance, "bucket", "spam") Helper.set_property(instance, "src_pattern", "spam") Helper.set_property(instance, "dest_dir", tmp_dir) instance.execute() assert m_get_object.call_args_list == []
def test_logging_mask_aws_keys(self): """ In log file, 'access_key' and 'secret_key' of AWS are masked. """ instance = SampleCustomStep() Helper.set_property(instance, "logger", LisboaLog.get_logger(instance.__class__.__name__)) Helper.set_property(instance, "access_key", "test") Helper.set_property(instance, "secret_key", "test") instance.trigger() masked_access_key = False masked_secret_key = False with open(self._log_file, mode="r", encoding="utf-8") as f: for line in f: if "access_key : ****" in line: masked_access_key = True elif "secret_key : ****" in line: masked_secret_key = True self.assertTrue(masked_access_key) self.assertTrue(masked_secret_key)
def _create_instance(self, pattern, refresh): instance = SqliteWrite() Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__)) Helper.set_property(instance, "dbname", self.DB_NAME) Helper.set_property(instance, "src_dir", ".") Helper.set_property(instance, "src_pattern", pattern) Helper.set_property(instance, "tblname", self.TBL_NAME) Helper.set_property(instance, "refresh", refresh) return instance