def test_execute_ok3(self):
try:
os.makedirs(self._data_dir, exist_ok=True)
# create test file
csv_list1 = [["key", "data"], ["c1", "spam"], ["c2", "spam"]]
with open(os.path.join(self._data_dir, "test1.csv"), "w") as t1:
writer = csv.writer(t1)
writer.writerows(csv_list1)
# set the essential attributes
instance = CsvConcat()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test.*\.csv")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", "test.csv")
instance.execute()
with open(os.path.join(self._data_dir, "test.csv")) as t:
reader = csv.reader(t)
concatenated_list = [row for row in reader]
finally:
shutil.rmtree(self._data_dir)
assert concatenated_list == [
["key", "data"],
["c1", "spam"],
["c2", "spam"],
]
def test_end_with_noerror(self):
if sys.version_info.minor < 6:
# ignore test if python version is less 3.6(assert_called is not supported)
return
with ExitStack() as stack:
mock_before_step = stack.enter_context(
patch("cliboa.core.listener.StepStatusListener.before_step"))
mock_error_step = stack.enter_context(
patch("cliboa.core.listener.StepStatusListener.error_step"))
mock_after_step = stack.enter_context(
patch("cliboa.core.listener.StepStatusListener.after_step"))
mock_post_step = stack.enter_context(
patch(
"cliboa.core.listener.StepStatusListener.after_completion")
)
step = SampleCustomStep()
Helper.set_property(step, "logger",
LisboaLog.get_logger(step.__class__.__name__))
Helper.set_property(step, "listeners", [StepStatusListener()])
executor = SingleProcExecutor([step])
executor.execute_steps(None)
mock_before_step.assert_called_once()
mock_error_step.assert_not_called()
mock_after_step.assert_called_once()
mock_post_step.assert_called_once()
def test_source_path_reader_with_none(self):
instance = SampleCustomStep()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(instance.__class__.__name__))
ret = instance._source_path_reader(None)
assert ret is None
def test_multi_process_error_stop(self):
py_info = sys.version_info
major_ver = py_info[0]
minor_ver = py_info[1]
py_ver = int(str(major_ver) + str(minor_ver))
log = LisboaLog.get_logger(self.__class__.__name__)
log.info(minor_ver)
if py_ver >= self.MULTI_PROC_SUPPORT_VER:
step1 = SampleStep()
Helper.set_property(step1, "logger",
LisboaLog.get_logger(step1.__class__.__name__))
step2 = ErrorSampleStep()
Helper.set_property(step2, "logger",
LisboaLog.get_logger(step2.__class__.__name__))
q = StepQueue()
q.force_continue = False
setattr(ScenarioQueue, "step_queue", q)
executor = MultiProcExecutor([step1, step2])
try:
executor.execute_steps(None)
self.fail("Error must be occured")
except StepExecutionFailed:
pass
def test_execute_ok_with_remain_column_numbers(self):
# create test csv
os.makedirs(self._data_dir, exist_ok=True)
test_csv = os.path.join(self._data_dir, "test.csv")
test_csv_data = [["1", "spam"], ["2", "spam"]]
with open(test_csv, "w") as t:
writer = csv.writer(t)
writer.writerows(test_csv_data)
t.flush()
# set the essential attributes
instance = CsvColumnExtract()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", "test.csv")
Helper.set_property(instance, "dest_dir", self._data_dir)
remain_column_number = 1
Helper.set_property(instance, "column_numbers", remain_column_number)
try:
instance.execute()
output_file = os.path.join(self._data_dir, "test.csv")
with open(output_file, "r") as o:
reader = csv.DictReader(o)
for r in reader:
assert r[test_csv_data[0][0]] == test_csv_data[1][0]
finally:
shutil.rmtree(self._data_dir)
def __create_di_instance(self, cls_attrs):
"""
Create an instance to be injected
Args:
step class attributes
Retrurn:
DI attribute names, DI instances
"""
di_keys = []
di_instance = None
di_instances = []
di_params = None
for k in cls_attrs.keys():
if k in self.DI_KEYS:
di_keys.append(k)
di_params = cls_attrs.get(k)
valid = DIScenarioFormat(k, di_params)
valid()
di_cls = di_params["class"]
di_cls = globals()[di_cls]
di_instance = di_cls()
if di_instance:
self._logger.debug(
"An instance %s to be injected exists." % di_instance
)
del di_params["class"]
# set attributes to instance
if di_params:
for k, v in di_params.items():
Helper.set_property(di_instance, k, v)
di_instances.append(di_instance)
return di_keys, di_instances
def test_source_path_reader_with_content(self):
instance = SampleCustomStep()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(instance.__class__.__name__))
ret = instance._source_path_reader({"content": "test"})
with open(ret, "r") as fp:
actual = fp.read()
assert "test" == actual
def __create_instance(self, s_dict, yaml_scenario_list):
cls_name = s_dict["class"]
self._logger.debug("Create %s instance" % cls_name)
if self.__is_custom_cls(cls_name) is True:
from cliboa.core.factory import CustomInstanceFactory
instance = CustomInstanceFactory.create(cls_name)
else:
cls = globals()[cls_name]
instance = cls()
cls_attrs_dict = {}
if isinstance(yaml_scenario_list, list) and "arguments" in s_dict.keys():
cls_attrs_dict = s_dict["arguments"]
# loop and set class attribute
di_key = None
di_instance = None
values = {}
if cls_attrs_dict:
cls_attrs_dict = self.__extract_with_vars(cls_attrs_dict)
# check if the keys of dependency injection
di_keys, di_instances = self.__create_di_instance(cls_attrs_dict)
if di_keys and di_instances:
di_keys_and_instances = zip(di_keys, di_instances)
for di_key, di_instance in di_keys_and_instances:
self._logger.debug(
"Inject %s to %s object." % (di_instance, instance)
)
# Injection
setattr(instance, di_key, di_instance)
del cls_attrs_dict[di_key]
pattern = re.compile(r"{{(.*?)}}")
for yaml_k, yaml_v in cls_attrs_dict.items():
# if value includes {{ var }}, replace value specified by with_vars
if isinstance(yaml_v, str):
matches = pattern.findall(yaml_v)
for match in matches:
var_name = match.strip()
yaml_v = self.__replace_vars(yaml_v, var_name)
Helper.set_property(instance, yaml_k, yaml_v)
values[yaml_k] = yaml_v
base_args = ["step", "symbol", "parallel", "io", "listeners"]
for arg in base_args:
if arg == "listeners":
self._append_listeners(instance, s_dict.get(arg), values)
else:
Helper.set_property(instance, arg, s_dict.get(arg))
return instance
def _append_listeners(self, instance, args):
listeners = [StepStatusListener()]
if args is not None:
from cliboa.core.factory import CustomInstanceFactory
if type(args) is str:
listeners.append(CustomInstanceFactory.create(args))
elif type(args) is list:
for arg in args:
listeners.append(CustomInstanceFactory.create(arg))
Helper.set_property(instance, "listeners", listeners)
def test_excute_ng_input_files(self):
with pytest.raises(InvalidCount) as execinfo:
# set the essential attributes
instance = CsvConcat()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_filenames", ["test1.csv"])
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", "test.csv")
instance.execute()
assert "Two or more input files are required." in str(execinfo.value)
def test_execute_ng_invalid_dbname(self):
"""
sqlite db does not exist.
"""
try:
instance = BaseSqlite()
db_file = os.path.join(self._db_dir, "spam.db")
Helper.set_property(instance, "dbname", db_file)
Helper.set_property(instance, "tblname", "spam_table")
instance.execute()
except Exception as e:
tb = sys.exc_info()[2]
assert "not found" in "{0}".format(e.with_traceback(tb))
def test_convert_header(self):
try:
# create test file
csv_list = [["key", "data"], ["1", "spam"], ["2", "spam"],
["3", "spam"]]
os.makedirs(self._data_dir, exist_ok=True)
test_csv = os.path.join(self._data_dir, "test.csv")
with open(test_csv, "w") as t:
writer = csv.writer(t)
writer.writerows(csv_list)
# set the essential attributes
instance = CsvConvert()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test\.csv")
Helper.set_property(instance, "headers", [{
"key": "new_key"
}, {
"data": "new_data"
}])
instance.execute()
with open(test_csv, "r") as t:
reader = csv.reader(t)
line = next(reader)
finally:
shutil.rmtree(self._data_dir)
assert line == ["new_key", "new_data"]
def test_ok(self):
if sys.version_info.minor < 6:
# ignore test if python version is less 3.6(assert_called is not supported)
return
step = SampleCustomStep()
Helper.set_property(step, "logger",
LisboaLog.get_logger(step.__class__.__name__))
clz = CustomStepListener()
values = {"test_key": "test_value"}
clz.__dict__.update(values)
Helper.set_property(step, "listeners", [clz])
executor = SingleProcExecutor([step])
executor.execute_steps(None)
def _create_instance(self):
instance = SqliteExport()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "dbname", self._DB_NAME)
Helper.set_property(instance, "dest_path", self._RESULT_FILE)
Helper.set_property(instance, "tblname", self._TBL_NAME)
return instance
def __invoke_steps(self, yaml_scenario_list):
"""
Create executable instance and push them to queue
Args:
yaml_scenario_list: parsed yaml list
"""
self._logger.info("Start to invoke scenario")
# Create queue to save step instances
q = StepQueue()
for s_dict in yaml_scenario_list:
if "multi_process_count" in s_dict.keys():
q.multi_proc_cnt = s_dict.get("multi_process_count")
continue
if "force_continue" in s_dict.keys():
q.force_continue = s_dict.get("force_continue")
continue
instances = []
if "parallel" in s_dict.keys():
for row in s_dict.get("parallel"):
instance = self.__create_instance(row, yaml_scenario_list)
Helper.set_property(
instance,
"logger",
LisboaLog.get_logger(instance.__class__.__name__),
)
instances.append(instance)
StepArgument._put(row["step"], instance)
else:
instance = self.__create_instance(s_dict, yaml_scenario_list)
Helper.set_property(
instance,
"logger",
LisboaLog.get_logger(instance.__class__.__name__),
)
instances.append(instance)
StepArgument._put(s_dict["step"], instance)
# Put instance to queue
q.push(instances)
# save queue to static area
setattr(ScenarioQueue, "step_queue", q)
self._logger.info("Finish to invoke scenario")
def _append_listeners(self, instance, args, values):
listeners = [StepStatusListener()]
if args is not None:
from cliboa.core.factory import CustomInstanceFactory
if type(args) is str:
clz = CustomInstanceFactory.create(args)
clz.__dict__.update(values)
listeners.append(clz)
elif type(args) is list:
for arg in args:
clz = CustomInstanceFactory.create(arg)
clz.__dict__.update(values)
listeners.append(CustomInstanceFactory.create(clz))
Helper.set_property(instance, "listeners", listeners)
def test_execute_ok(self):
try:
os.makedirs(self._data_dir)
instance = HttpDownload()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
# use Postman echo
Helper.set_property(instance, "src_url", "https://postman-echo.com")
Helper.set_property(instance, "src_pattern", "get?foo1=bar1&foo2=bar2")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", "test.result")
instance.execute()
f = open(os.path.join(self._data_dir, "test.result"), "r")
result = f.read()
f.close()
finally:
shutil.rmtree(self._data_dir)
assert "postman-echo.com" in result
def test_logging_mask_password(self):
"""
In log file, 'password' is masked.
"""
instance = SampleCustomStep()
instance._logger = LisboaLog.get_logger(__name__)
Helper.set_property(instance, "logger",
LisboaLog.get_logger(instance.__class__.__name__))
Helper.set_property(instance, "password", "test")
instance.trigger()
ret = False
with open(self._log_file, mode="r", encoding="utf-8") as f:
for line in f:
if "password : ****" in line:
ret = True
break
self.assertTrue(ret)
def test_ok_5(self):
"""
Input csv file is only one.
Sqlite database already exists.
Csv columns and table columns are not the same.
force_insert is True.
"""
TEST_FILE_1 = "sqlite_write_test_1.csv"
TEST_FILE_2 = "sqlite_write_test_2.csv"
try:
with open(TEST_FILE_1, "w", encoding="utf-8", newline="") as f:
writer = csv.writer(f)
writer.writerow(["No", "TEXT"])
writer.writerow(["1", "A"])
with open(TEST_FILE_2, "w", encoding="utf-8", newline="") as f:
writer = csv.writer(f)
writer.writerow(["No", "NAME"])
writer.writerow(["2", "John"])
instance = self._create_instance(TEST_FILE_1, True)
instance.execute()
instance = self._create_instance(TEST_FILE_2, False)
Helper.set_property(instance, "force_insert", True)
instance.execute()
adapter = SqliteAdapter()
adapter.connect(self.DB_NAME)
cur = adapter.fetch(
"SELECT * FROM %s" % self.TBL_NAME, row_factory=self._dict_factory
)
count = 0
for row in cur:
if count == 0:
assert row == {"No": "1", "TEXT": "A", "NAME": None}
elif count == 1:
assert row == {"No": "2", "TEXT": None, "NAME": "John"}
count += 1
finally:
self._clean(self.DB_NAME)
self._clean(TEST_FILE_1)
self._clean(TEST_FILE_2)
def test_source_path_reader_with_path(self):
try:
os.makedirs(self._data_dir)
dummy_pass = os.path.join(self._data_dir, "id_rsa")
with open(dummy_pass, "w") as f:
f.write("test")
instance = SampleCustomStep()
Helper.set_property(
instance, "logger",
LisboaLog.get_logger(instance.__class__.__name__))
ret = instance._source_path_reader({"file": dummy_pass})
assert ret == dummy_pass
with open(ret, "r") as fp:
actual = fp.read()
assert "test" == actual
finally:
shutil.rmtree(self._data_dir)
def test_ok_3(self):
"""
Insert plural records(some records are the same)
to a table and export the table with no_duplicate option.
"""
test_data = [
{
"No": 1,
"TEXT": "AAA"
},
{
"No": 2,
"TEXT": "BBB"
},
{
"No": 2,
"TEXT": "BBB"
},
]
try:
self._insert_test_data(test_data)
instance = self._create_instance()
Helper.set_property(instance, "order", ["No"])
Helper.set_property(instance, "no_duplicate", True)
instance.execute()
with open(self._RESULT_FILE, "r") as o:
reader = csv.DictReader(o)
for i, row in enumerate(reader):
if i == 0:
assert "1" == row.get("No")
assert "AAA" == row.get("TEXT")
elif i == 1:
assert "2" == row.get("No")
assert "BBB" == row.get("TEXT")
else:
pytest.fail("Must not be reached")
finally:
self._clean(self._DB_NAME)
self._clean(self._RESULT_FILE)
def test_multi_process_error_continue(self):
py_info = sys.version_info
major_ver = py_info[0]
minor_ver = py_info[1]
py_ver = int(str(major_ver) + str(minor_ver))
if py_ver >= self.MULTI_PROC_SUPPORT_VER:
step1 = SampleStep()
Helper.set_property(step1, "logger",
LisboaLog.get_logger(step1.__class__.__name__))
step2 = ErrorSampleStep()
Helper.set_property(step2, "logger",
LisboaLog.get_logger(step2.__class__.__name__))
q = StepQueue()
q.force_continue = True
setattr(ScenarioQueue, "step_queue", q)
executor = MultiProcExecutor([step1, step2])
executor.execute_steps(None)
def test_sort(self):
try:
# create test file
csv_list = [["key", "data"], ["1", "A"], ["3", "C"], ["2", "B"]]
result_dir = os.path.join(self._data_dir, "result")
os.makedirs(result_dir, exist_ok=True)
test_csv1 = os.path.join(self._data_dir, "test1.csv")
with open(test_csv1, "w") as t:
writer = csv.writer(t)
writer.writerows(csv_list)
test_csv2 = os.path.join(self._data_dir, "test2.csv")
with open(test_csv2, "w") as t:
writer = csv.writer(t)
writer.writerows(csv_list)
# set the essential attributes
instance = CsvSort()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test.*\.csv")
Helper.set_property(instance, "dest_dir", result_dir)
Helper.set_property(instance, "order", ["key"])
instance.execute()
files = glob(os.path.join(result_dir, "test*.csv"))
assert 2 == len(files)
for file in files:
with open(file, mode='r', encoding="utf-8") as f:
reader = csv.DictReader(f)
for i, row in enumerate(reader):
if i == 0:
assert "1" == row.get("key")
elif i == 1:
assert "2" == row.get("key")
elif i == 2:
assert "3" == row.get("key")
finally:
shutil.rmtree(self._data_dir)
def test_execute_ok(self):
try:
# create test file
os.makedirs(self._data_dir, exist_ok=True)
excel_file = os.path.join(self._data_dir, "test.xlxs")
workbook = xlsxwriter.Workbook(excel_file)
workbook.close()
# set the essential attributes
instance = ExcelConvert()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test\.xlxs")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", "test.csv")
instance.execute()
exists_csv = glob(os.path.join(self._data_dir, "test.csv"))
finally:
shutil.rmtree(self._data_dir)
assert "test.csv" in exists_csv[0]
def test_excute_ng_multiple_src(self):
with pytest.raises(InvalidCount) as execinfo:
try:
# create test file
os.makedirs(self._data_dir, exist_ok=True)
excel_file = os.path.join(self._data_dir, "test1.xlxs")
open(excel_file, "w").close()
excel_file2 = os.path.join(self._data_dir, "test2.xlxs")
open(excel_file2, "w").close()
# set the essential attributes
instance = ExcelConvert()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test(.*)\.xlxs")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", r"test(.*).xlxs")
instance.execute()
finally:
shutil.rmtree(self._data_dir)
assert "must be only one" in str(execinfo.value)
def test_execute_ng_invalid_extension(self):
with pytest.raises(InvalidFormat) as execinfo:
try:
# create test file
os.makedirs(self._data_dir, exist_ok=True)
excel_file = os.path.join(self._data_dir, "test.xlxs")
open(excel_file, "w").close()
excel_file2 = os.path.join(self._data_dir, "test.xlxs.bk")
open(excel_file2, "w").close()
# set the essential attributes
instance = ExcelConvert()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", "test.xlxs")
Helper.set_property(instance, "dest_dir", self._data_dir)
Helper.set_property(instance, "dest_pattern", "test.xlxs")
instance.execute()
finally:
shutil.rmtree(self._data_dir)
assert "not supported" in str(execinfo.value)
def test_compress_zip(self):
try:
# create test file
os.makedirs(self._data_dir, exist_ok=True)
test_file = os.path.join(self._data_dir, "test.txt")
with open(test_file, "w") as t:
t.write("ABCDEF")
# set the essential attributes
instance = FileArchive()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "src_dir", self._data_dir)
Helper.set_property(instance, "src_pattern", r"test\.txt")
Helper.set_property(instance, "format", "zip")
Helper.set_property(instance, "dest_pattern", "foo")
instance.execute()
files = glob(os.path.join(self._data_dir, "*.zip"))
assert 1 == len(files)
assert "foo.zip" == os.path.basename(files[0])
finally:
shutil.rmtree(self._data_dir)
def test_execute_ok(self, m_get_client):
with tempfile.TemporaryDirectory() as tmp_dir:
m_get_object = m_get_client.return_value.get_object
m_pagenate = m_get_client.return_value.get_paginator.return_value.paginate
m_contents = [{"Contents": [{"Key": "spam"}]}]
m_pagenate.return_value = m_contents
instance = S3Download()
Helper.set_property(instance, "bucket", "spam")
Helper.set_property(instance, "src_pattern", "spam")
Helper.set_property(instance, "dest_dir", tmp_dir)
instance.execute()
assert m_get_object.call_args_list == []
def test_logging_mask_aws_keys(self):
"""
In log file, 'access_key' and 'secret_key' of AWS are masked.
"""
instance = SampleCustomStep()
Helper.set_property(instance, "logger",
LisboaLog.get_logger(instance.__class__.__name__))
Helper.set_property(instance, "access_key", "test")
Helper.set_property(instance, "secret_key", "test")
instance.trigger()
masked_access_key = False
masked_secret_key = False
with open(self._log_file, mode="r", encoding="utf-8") as f:
for line in f:
if "access_key : ****" in line:
masked_access_key = True
elif "secret_key : ****" in line:
masked_secret_key = True
self.assertTrue(masked_access_key)
self.assertTrue(masked_secret_key)
def _create_instance(self, pattern, refresh):
instance = SqliteWrite()
Helper.set_property(instance, "logger", LisboaLog.get_logger(__name__))
Helper.set_property(instance, "dbname", self.DB_NAME)
Helper.set_property(instance, "src_dir", ".")
Helper.set_property(instance, "src_pattern", pattern)
Helper.set_property(instance, "tblname", self.TBL_NAME)
Helper.set_property(instance, "refresh", refresh)
return instance
