def moveProcessedFiles(self): json_list = self.hash_json.data() for key in json_list: source_path = self.storage.hd2_data(key) if (FileStatus.COMPLETED == json_list[key]["file_status"]): destination_path = self.storage.hd2_processed(key) if folder_exists(destination_path): folder_delete_all(destination_path) shutil.move(source_path, destination_path) if (FileStatus.FAILED == json_list[key]["file_status"]): meta_service = Metadata_Service() meta_service.get_from_file(source_path) metadata = meta_service.metadata if ("Engine response could not be decoded" == metadata.get_error()) and \ metadata.get_original_file_extension() in ['.xml', '.json']: destination_path = self.storage.hd2_not_processed(key) if folder_exists(destination_path): folder_delete_all(destination_path) shutil.move(source_path, destination_path)
def test__init__(self): self.pre_processor.clear_data_and_status_folders( ) # clear output folders self.pre_processor.process_files() # copy files across assert folder_exists(self.config.hd1_location) assert folder_exists(self.config.hd2_location) assert folder_exists(self.config.hd3_location)
def test_folder_create(self): tmp_folder = '_tmp_folder' assert folder_exists(tmp_folder) is False assert folder_create(tmp_folder) == tmp_folder assert folder_create(tmp_folder) == tmp_folder assert folder_exists(tmp_folder) is True assert folder_not_exists(tmp_folder) is False assert folder_delete_all(tmp_folder) is True assert folder_not_exists(tmp_folder) is True
def test_folder_create_in_parent(self): tmp_folder = '_tmp_folder' child_folder = '_child_folder' assert folder_exists(tmp_folder) == False assert folder_create(tmp_folder) == tmp_folder assert create_folder_in_parent(tmp_folder, child_folder) == path_combine( tmp_folder, child_folder) assert folder_exists(path_combine(tmp_folder, child_folder)) == True assert folder_delete_all(tmp_folder) == True assert folder_not_exists(path_combine(tmp_folder, child_folder)) == True
def metadata_folder_path(self): if not self.file_hash: return path = self.storage.hd2_not_processed(self.file_hash) if folder_exists(path): return path path = self.storage.hd2_processed(self.file_hash) if folder_exists(path): return path # never processed - must be in the 'todo' folder path = self.storage.hd2_data(self.file_hash) return path
def test_get_package(self): package = self.deploy.get_package() assert package.lambda_name == 'osbot_test_deploy_lambda.osbot_test_deploy_lambda' assert package.s3_bucket == self.aws_config.lambda_s3_bucket() assert package.s3_key == f'{self.aws_config.lambda_s3_folder_lambdas()}/{package.lambda_name}.zip' assert package.role_arn == f"arn:aws:iam::{self.aws_config.aws_session_account_id()}:role/temp_role_for_lambda_invocation" assert folder_exists(package.tmp_folder)
def test_copy_file(self): assert file_exists(self.test_file) assert folder_exists(self.new_folder) self.dst = os.path.join(self.new_folder, "image2.jpg") self.file_service.copy_file(self.test_file, self.dst) assert os.path.exists(self.dst) is True
def pip_install_dependency(target): path_lambda_dependencies = Files.path_combine('.', '../../../_lambda_dependencies/') folder_create(path_lambda_dependencies) path_install = Files.path_combine(path_lambda_dependencies, target) if folder_not_exists(path_install): return Process.run('pip3', ['install','-t',path_install,target]) return folder_exists(path_install)
def test_file_create_and_deletion(self): with Temp_S3_Zip_With_Lambda_File() as temp_s3_zip: assert folder_exists(temp_s3_zip.folder) assert file_exists(temp_s3_zip.tmp_file) assert temp_s3_zip.s3_prefix == 'lambdas/unit_tests/temp_zips' assert 'temp_zip_file_' in temp_s3_zip.file_name assert temp_s3_zip.s3_key == f'{temp_s3_zip.s3_prefix}/{temp_s3_zip.file_name}.zip'
def test_image_build(self): target_image = 'centos' expected_size = 209348126 folder_dockerFile = path_combine(self.path_docker_images, target_image) path_dockerfile = path_combine(folder_dockerFile, 'Dockerfile') repository = "osbot_docker__test_image_build" tag = "abc" image_name = f"{repository}:{tag}" assert folder_exists(folder_dockerFile) assert file_exists(path_dockerfile) result = self.api_docker.image_build(folder_dockerFile, repository, tag) build_logs = result.get('build_logs') image = result.get('image') status = result.get('status') tags = result.get('tags') assert self.api_docker.image_exists(repository, tag) assert status == 'ok' assert image_name in tags assert image_name in self.api_docker.images_names() assert image.get('Size') == expected_size assert next(build_logs) == {'stream': 'Step 1/3 : FROM centos:8'} assert self.api_docker.image_delete(repository, tag) is True assert image_name not in self.api_docker.images_names()
def test_add_file(self): metadata = self.metadata file_paths = metadata.data.get('original_file_paths') assert self.metadata.exists() is False # metadata folder doesn't exist # adding file first time assert metadata.add_file( self.file_path ) == self.file_hash # add file and get file hash as return value assert metadata.exists() is True # confirm metadata folder now exists assert folder_exists(metadata.metadata_folder_path() ) # confirm metadata folder now exists assert file_exists( metadata.metadata_file_path()) # confirm metadata json file exists assert file_exists(metadata.source_file_path() ) # confirm source file was correctly put in place assert metadata.file_hash == self.metadata_utils.file_hash( metadata.source_file_path() ) # confirm hash of source file matches hash of file_path assert metadata.metadata_file_path() == path_combine( metadata.metadata_folder_path(), DEFAULT_METADATA_FILENAME ) # confirm metadata file is place in correct location file_paths = metadata.data.get('original_file_paths') assert file_paths == [ self.file_path ] # confirms that in this mode the entire path is preserved # adding same file 2nd time (with same hash and same name) assert metadata.add_file( self.file_path) == self.file_hash # adding the same file again file_paths = metadata.data.get('original_file_paths') assert file_paths == [self.file_path ] # should not impact this value (same as above) # adding same file 3nd time (with same hash but different name) assert metadata.add_file( self.file_copy_path ) == self.file_hash # adding the same file again (with different name) file_paths = metadata.data.get('original_file_paths') assert file_paths == [self.file_path, self.file_copy_path ] # will make the new file path be added # adding same file 4th time (with self.path_hd1 set to parent folder of path) file_parent_folder = parent_folder( self.file_path) # get parent folder of test file self.metadata.path_hd1 = file_parent_folder # assign it to the metadata variable used to calculate virtual paths assert metadata.add_file(self.file_path) == self.file_hash file_paths = metadata.data.get('original_file_paths') assert file_paths == [ self.file_path, self.file_copy_path, file_name(self.file_path) ] # confirm that the virtual file path was added as the 3rd item (in this case the file name) #clean up assert self.metadata.delete() is True assert folder_not_exists(self.metadata.metadata_folder_path())
def test_set_root_folder(self): root_folder = temp_folder() assert self.config.root_folder != root_folder self.config.set_root_folder(root_folder) assert self.config.root_folder == root_folder assert self.config.hd1_location == path_combine( root_folder, DEFAULT_HD1_NAME) assert self.config.hd2_location == path_combine( root_folder, DEFAULT_HD2_NAME) assert self.config.hd3_location == path_combine( root_folder, DEFAULT_HD3_NAME) assert folder_exists(self.config.root_folder) assert folder_exists(self.config.hd1_location) assert folder_exists(self.config.hd2_location) assert folder_exists(self.config.hd2_data_location) assert folder_exists(self.config.hd2_status_location) assert folder_exists(self.config.hd2_processed_location) assert folder_exists(self.config.hd3_location) folder_delete_all(root_folder) assert folder_not_exists(self.config.root_folder) assert folder_not_exists(self.config.hd1_location) assert folder_not_exists(self.config.hd2_location) assert folder_not_exists(self.config.hd3_location)
def test_moveProcessedFiles(self): Loops.continue_processing = True count = 20 self.add_test_files(count=count, execute_stage_1=True) json_data = self.loops.updateHashJson() assert len(json_data) > 0 self.loops.moveProcessedFiles() assert folder_exists(self.loops.storage.hd2_processed())
def test_set_config_to_temp_folder__restore_config(self): storage = Storage() config = storage.config original_config = config.values() self.setup_testing.set_config_to_temp_folder() temp_config = config.values() assert parent_folder(config.root_folder ) == temp_folder_current() assert folder_exists(config.root_folder ) assert folder_exists(storage.hd1() ) assert folder_exists(storage.hd2_status()) assert folder_exists(storage.hd2_data() ) assert folder_exists(storage.hd3() ) assert original_config != temp_config self.setup_testing.restore_config() #self.setup_testing.configure_static_logging() assert original_config == config.values() assert parent_folder(config.root_folder) != temp_folder_current() assert folder_not_exists(temp_config.get('root_folder'))
def test_invalid_hd3(self): hd1_path = "./test_data/scenario-1/hd1" hd2_path = "./test_data/scenario-1/hd2" hd3_path = "./test_data/scenario-1/hd3xyz" response=self.configure.configure(hd1_path=hd1_path, hd2_path=hd2_path, hd3_path=hd3_path) assert self.configure.last_error_message == "" assert response is not None assert folder_exists(hd3_path) folder_delete_all(hd3_path)
def test_load_values(self): config = self.config self.assertEqual( abspath(config.hd1_location), abspath(os.environ.get("HD1_LOCATION", DEFAULT_HD1_LOCATION))) self.assertEqual( abspath(config.hd2_location), abspath(os.environ.get("HD2_LOCATION", DEFAULT_HD2_LOCATION))) self.assertEqual( abspath(config.hd3_location), abspath(os.environ.get("HD3_LOCATION", DEFAULT_HD3_LOCATION))) self.assertEqual( abspath(config.root_folder), abspath(os.environ.get("ROOT_FOLDER", DEFAULT_ROOT_FOLDER))) self.assertEqual( config.endpoints, json.loads(os.environ.get("ENDPOINTS", DEFAULT_ENDPOINTS))) assert config.endpoints['Endpoints'][0]['IP'] assert config.endpoints['Endpoints'][0]['Port'] assert folder_exists(config.root_folder) assert folder_exists(config.hd1_location) assert folder_exists(config.hd2_location) assert folder_exists(config.hd2_data_location) assert folder_exists(config.hd2_status_location) assert folder_exists(config.hd2_processed_location) assert folder_exists(config.hd3_location) # check config_cache config.root_folder = 'aaa' assert Config().root_folder == 'aaa' config.load_values() assert config.root_folder == DEFAULT_ROOT_FOLDER assert Config().root_folder == DEFAULT_ROOT_FOLDER
def LoopHashDirectoriesInternal(self, thread_count, do_single): if folder_exists(self.storage.hd2_data()) is False: log_message = "ERROR: rootdir does not exist: " + self.storage.hd2_data( ) log_error(log_message) return False if not isinstance(thread_count, int): raise TypeError("thread_count must be a integer") if not isinstance(do_single, bool): raise TypeError("thread_count must be a integer") log_message = f"LoopHashDirectoriesInternal started with {thread_count} threads" self.events.add_log(log_message) log_info(log_message) json_list = self.updateHashJson() log_message = f"LoopHashDirectoriesInternal started with {thread_count} threads" self.events.add_log(log_message) log_info(log_message) threads = list() process_index = 0 log_info( message=f'before Mapping thread_data for {len(json_list)} files') thread_data = [] for key in json_list: file_hash = key itempath = self.storage.hd2_data(key) if (FileStatus.COMPLETED == json_list[key]["file_status"]): self.events.add_log( f"The file processing has been already completed") continue if not os.path.exists(itempath): self.events.add_log( f"ERROR: Path \"{itempath}\" does not exist") json_list[key]["file_status"] = FileStatus.FAILED continue process_index += 1 thread_data.append(( itempath, file_hash, process_index, )) # # limit the number of parallel threads # # if process_index % int(thread_count) == 0: # todo: refactor this workflow to use multiprocess and queues # # Clean up the threads # for index, thread in enumerate(threads): # todo: since at the moment this will block allocating new threads until # thread.join() # all have finishing execution # # process_index += 1 # log_info(message=f"in LoopHashDirectoriesInternal process_index={process_index} , thread #{process_index % int(thread_count) }") # x = threading.Thread(target=self.ProcessDirectory, args=(itempath, file_hash, process_index,)) # threads.append(x) # x.start() # # if do_single: # break # # if not Loops.continue_processing: # break # for index, thread in enumerate(threads): # thread.join() log_info( message= f'after mapped thread_data, there are {len(thread_data)} mapped items' ) #thread_data = thread_data[:500] #log_info(message=f'to start with only processing {len(thread_data)} thread_data items') pool = ThreadPool(thread_count) results = pool.map(self.ProcessDirectory, thread_data) pool.close() pool.join() self.moveProcessedFiles() self.events.add_log("LoopHashDirectoriesInternal finished") return True
def test_path_images(self): assert folder_exists(self._.path_images)
def test_icons_folder(self): assert folder_exists(self.jira_icons.icons_folder())
def test__init__(self): assert folder_exists(self.test_data.path_test_files)
def test_set_test_root_dir(self): self.setup_testing.set_test_root_dir() assert folder_exists(path_combine('.', '.git')) assert folder_exists(path_combine('.', 'test_data'))
def exists(self): return folder_exists(self.metadata_folder_path())
def test_path_folder_with_docker_file(self): path = self.icap_client.path_folder_with_docker_file() dockerfile = path_combine(path, 'Dockerfile') assert folder_exists(path) assert file_exists(dockerfile)
def test_path_repo_root(self): path_repo = self.setup_testing.path_repo_root() assert folder_exists(path_repo) assert folder_exists(path_combine(path_repo, '.git')) assert folder_exists(path_combine(path_repo, 'test_data'))
def is_in_todo(self): folder_exists(self.storage.hd2_data(self.file_hash))
def is_in_not_processed(self): folder_exists(self.storage.hd2_not_processed(self.file_hash))
def exists_locally(self, package): return folder_exists(self.local_path(package))
def test__init__(self): assert folder_exists(self.pre_processor.storage.hd2_data()) assert folder_exists(self.pre_processor.storage.hd2_status())