def test_set_root_folder(self):
root_folder = temp_folder()
assert self.config.root_folder != root_folder
self.config.set_root_folder(root_folder)
assert self.config.root_folder == root_folder
assert self.config.hd1_location == path_combine(
root_folder, DEFAULT_HD1_NAME)
assert self.config.hd2_location == path_combine(
root_folder, DEFAULT_HD2_NAME)
assert self.config.hd3_location == path_combine(
root_folder, DEFAULT_HD3_NAME)
assert folder_exists(self.config.root_folder)
assert folder_exists(self.config.hd1_location)
assert folder_exists(self.config.hd2_location)
assert folder_exists(self.config.hd2_data_location)
assert folder_exists(self.config.hd2_status_location)
assert folder_exists(self.config.hd2_processed_location)
assert folder_exists(self.config.hd3_location)
folder_delete_all(root_folder)
assert folder_not_exists(self.config.root_folder)
assert folder_not_exists(self.config.hd1_location)
assert folder_not_exists(self.config.hd2_location)
assert folder_not_exists(self.config.hd3_location)
def moveProcessedFiles(self):
json_list = self.hash_json.data()
for key in json_list:
source_path = self.storage.hd2_data(key)
if (FileStatus.COMPLETED == json_list[key]["file_status"]):
destination_path = self.storage.hd2_processed(key)
if folder_exists(destination_path):
folder_delete_all(destination_path)
shutil.move(source_path, destination_path)
if (FileStatus.FAILED == json_list[key]["file_status"]):
meta_service = Metadata_Service()
meta_service.get_from_file(source_path)
metadata = meta_service.metadata
if ("Engine response could not be decoded" == metadata.get_error()) and \
metadata.get_original_file_extension() in ['.xml', '.json']:
destination_path = self.storage.hd2_not_processed(key)
if folder_exists(destination_path):
folder_delete_all(destination_path)
shutil.move(source_path, destination_path)
def test_copy_folder(self):
temp_folder = folder_temp()
folder_delete_all(temp_folder)
self.file_service.copy_folder(self.test_folder, temp_folder)
directory = os.listdir(temp_folder)
assert len(directory) is not 0
def __init__(self):
self.config = Config()
self.hd1_base_location = self.config.hd1_location
self.hd2_base_location = self.config.hd2_location
self.hd3_base_location = self.config.hd3_location
self.zip_folder = os.path.join(os.getcwd(),"zip_folder")
self.storage = Storage()
folder_delete_all(self.zip_folder)
create_folder(self.zip_folder)
def prepare_folder(self, folder_to_process):
if folder_to_process.startswith(self.storage.hd1()):
return folder_to_process
dirname = os.path.join(self.storage.hd1(),
os.path.basename(folder_to_process))
if os.path.isdir(dirname):
folder_delete_all(dirname)
try:
folder_copy(folder_to_process, dirname)
finally:
return dirname
def test_invalid_hd3(self):
hd1_path = "./test_data/scenario-1/hd1"
hd2_path = "./test_data/scenario-1/hd2"
hd3_path = "./test_data/scenario-1/hd3xyz"
response=self.configure.configure(hd1_path=hd1_path,
hd2_path=hd2_path,
hd3_path=hd3_path)
assert self.configure.last_error_message == ""
assert response is not None
assert folder_exists(hd3_path)
folder_delete_all(hd3_path)
def test_folder_create(self):
tmp_folder = '_tmp_folder'
assert folder_exists(tmp_folder) is False
assert folder_create(tmp_folder) == tmp_folder
assert folder_create(tmp_folder) == tmp_folder
assert folder_exists(tmp_folder) is True
assert folder_not_exists(tmp_folder) is False
assert folder_delete_all(tmp_folder) is True
assert folder_not_exists(tmp_folder) is True
def test_set_hd3_location(self):
hd3_location = path_combine(temp_folder(), 'cccc')
assert file_not_exists(hd3_location)
self.config.set_hd3_location(hd3_location)
assert self.config.hd3_location == hd3_location
assert file_exists(hd3_location)
self.config.load_values()
assert abspath(
self.config.hd3_location) == abspath('./test_data/scenario-1/hd3')
assert folder_delete_all(hd3_location)
def prepare_zip(self, path, zip_name):
try:
self.temp_folder = temp_folder()
dst_path = os.path.join(self.temp_folder, ntpath.basename(path))
if os.path.isfile(path):
file_copy(path, dst_path)
elif os.path.isdir(path):
folder_copy(path, dst_path)
target_file_path = os.path.join(self.zip_folder, zip_name)
zip_files(self.temp_folder, file_pattern='*.*', target_file = target_file_path)
folder_delete_all(self.temp_folder)
return target_file_path
except Exception as error:
logger.error(f"File_Distributor: prepare_zip : {error}")
raise error
def test_folder_create_in_parent(self):
tmp_folder = '_tmp_folder'
child_folder = '_child_folder'
assert folder_exists(tmp_folder) == False
assert folder_create(tmp_folder) == tmp_folder
assert create_folder_in_parent(tmp_folder,
child_folder) == path_combine(
tmp_folder, child_folder)
assert folder_exists(path_combine(tmp_folder, child_folder)) == True
assert folder_delete_all(tmp_folder) == True
assert folder_not_exists(path_combine(tmp_folder,
child_folder)) == True
def test_process_folder(self):
path_data = self.pre_processor.storage.hd2_data()
filename = os.path.basename(self.test_file)
folder_to_process = self.pre_processor.prepare_folder(self.temp_dir)
assert folder_to_process.startswith(self.pre_processor.storage.hd1())
assert os.path.isdir(folder_to_process)
assert os.path.isfile(os.path.join(folder_to_process, filename))
folder_delete_all(folder_to_process)
assert not os.path.isfile(os.path.join(folder_to_process, filename))
self.pre_processor.clear_data_and_status_folders()
self.pre_processor.process_folder(self.temp_dir)
assert os.path.isdir(folder_to_process)
assert os.path.isfile(os.path.join(folder_to_process, filename))
assert len(files_list(path_data)) == 1
self.pre_processor.process_folder(self.temp_dir)
assert len(files_list(path_data)) == 1
folder_delete_all(folder_to_process)
self.pre_processor.clear_data_and_status_folders()
def prepare_hd2_hash_folder_zip(self,path_list, zip_name):
try:
self.temp_folder = temp_folder()
for hash_folder_path in path_list:
name = ntpath.basename(hash_folder_path)
dst_path = os.path.join(self.temp_folder, name)
if os.path.isdir(hash_folder_path):
folder_copy(hash_folder_path, dst_path)
hd2_source_file = os.path.join(dst_path, "source")
if os.path.isfile(hd2_source_file):
file_delete(hd2_source_file)
target_file_path = os.path.join(self.zip_folder, zip_name)
zip_files(self.temp_folder, file_pattern='*.*', target_file = target_file_path)
folder_delete_all(self.temp_folder)
return target_file_path
except Exception as error:
logger.error(f"File_Distributor: prepare_zip : {error}")
raise error
def test_set_hd2_location(self):
hd2_location = path_combine(temp_folder(), 'bbb')
assert file_not_exists(hd2_location)
self.config.set_hd2_location(hd2_location)
assert self.config.hd2_location == hd2_location
assert self.config.hd2_data_location == path_combine(
hd2_location, DEFAULT_HD2_DATA_NAME)
assert self.config.hd2_status_location == path_combine(
hd2_location, DEFAULT_HD2_STATUS_NAME)
assert self.config.hd2_processed_location == path_combine(
hd2_location, DEFAULT_HD2_PROCESSED_NAME)
assert file_exists(hd2_location)
self.config.load_values()
assert abspath(
self.config.hd2_location) == abspath('./test_data/scenario-1/hd2')
assert folder_delete_all(hd2_location)
def clear_data_and_status_folders(self):
data_target = self.storage.hd2_data(
) # todo: refactor this clean up to the storage class
status_target = self.storage.hd2_status()
processed_target = self.storage.hd2_processed()
folder_delete_all(data_target)
folder_delete_all(status_target)
folder_delete_all(processed_target)
folder_create(data_target)
folder_create(status_target)
folder_create(processed_target)
self.status.reset()
def hd2_delete_all_files(self):
folder_delete_all(self.hd2_data())
folder_delete_all(self.hd2_status())
folder_create(self.hd2_data())
folder_create(self.hd2_status())
def hd1_delete_all_files(self):
folder_delete_all(self.hd1())
folder_create(self.hd1())
def tearDownClass(cls) -> None:
cls.pre_processor.clear_data_and_status_folders()
folder_delete_all(os.path.join(os.getcwd(), "zip_folder"))
def delete(self):
if self.exists():
folder_delete_all(self.metadata_folder_path())
return self.exists() is False
return False
def tearDownClass(cls) -> None:
file_delete(cls.test_file)
folder_delete_all(cls.temp_dir)
Metadata(file_hash=cls.file_hash).delete()
def restore_config(self):
folder_delete_all(self.temp_root_folder)
self.get_config().load_values() # reload config values