def list_files(self, path):
if not self._is_recursive_server_file():
raise WrongParameterUsageException(self.name, 'Can list files only for recursive file parameters')
validation_error = self._validate_recursive_path(path, intermediate=True)
if validation_error:
raise InvalidValueException(self.name, validation_error)
full_path = self._build_list_file_path(path)
result = []
if is_empty(self.file_type) or self.file_type == FILE_TYPE_FILE:
files = model_helper.list_files(full_path, FILE_TYPE_FILE, self.file_extensions)
for file in files:
result.append({'name': file, 'type': FILE_TYPE_FILE, 'readable': True})
dirs = model_helper.list_files(full_path, FILE_TYPE_DIR)
for dir in dirs:
dir_path = os.path.join(full_path, dir)
readable = os.access(dir_path, os.R_OK)
result.append({'name': dir, 'type': FILE_TYPE_DIR, 'readable': readable})
return result
def list_files(self, path):
if not self._is_recursive_server_file():
raise WrongParameterUsageException(self.name, 'Can list files only for recursive file parameters')
validation_error = self._validate_recursive_path(path, intermediate=True)
if validation_error:
raise InvalidValueException(self.name, validation_error)
full_path = self._build_list_file_path(path)
result = []
if is_empty(self.file_type) or self.file_type == FILE_TYPE_FILE:
files = model_helper.list_files(full_path, FILE_TYPE_FILE, self.file_extensions)
for file in files:
result.append({'name': file, 'type': FILE_TYPE_FILE, 'readable': True})
dirs = model_helper.list_files(full_path, FILE_TYPE_DIR)
for dir in dirs:
dir_path = os.path.join(full_path, dir)
readable = os.access(dir_path, os.R_OK)
result.append({'name': dir, 'type': FILE_TYPE_DIR, 'readable': readable})
return result
def _validate_recursive_path(self, path, intermediate):
value_string = self.value_to_str(path)
if not isinstance(path, list):
return 'should be a list, but was: ' + value_string + '(' + str(
type(path)) + ')'
if ('.' in path) or ('..' in path):
return 'Relative path references are not allowed'
full_path = self._build_list_file_path(path)
if not os.path.exists(full_path):
return 'Path ' + value_string + ' does not exist'
if intermediate:
if not os.access(full_path, os.R_OK):
return 'Path ' + value_string + ' not accessible'
if not os.path.isdir(full_path):
return 'Path ' + value_string + ' is not a directory'
else:
dir = path[:-1]
file = path[-1]
dir_path = self._build_list_file_path(dir)
allowed_files = model_helper.list_files(dir_path, self.file_type,
self.file_extensions)
if file not in allowed_files:
return 'Path ' + value_string + ' is not allowed'
def test_file_type_dir(self):
files = ['file1', 'file2']
test_utils.create_files(files)
test_utils.create_dir('my_dir')
actual_files = model_helper.list_files(test_utils.temp_folder, file_type=FILE_TYPE_DIR)
self.assertEqual(['my_dir'], actual_files)
def _validate_recursive_path(self, path, intermediate):
value_string = self.value_to_str(path)
if not isinstance(path, list):
return 'should be a list, but was: ' + value_string + '(' + str(type(path)) + ')'
if ('.' in path) or ('..' in path):
return 'Relative path references are not allowed'
full_path = self._build_list_file_path(path)
if not os.path.exists(full_path):
return 'Path ' + value_string + ' does not exist'
if intermediate:
if not os.access(full_path, os.R_OK):
return 'Path ' + value_string + ' not accessible'
if not os.path.isdir(full_path):
return 'Path ' + value_string + ' is not a directory'
else:
dir = path[:-1]
file = path[-1]
dir_path = self._build_list_file_path(dir)
allowed_files = model_helper.list_files(dir_path, self.file_type, self.file_extensions)
if file not in allowed_files:
return 'Path ' + value_string + ' is not allowed'
def test_glob_relative_path(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files)
matcher = self.create_matcher(['*1'])
files = model_helper.list_files(test_utils.temp_folder,
excluded_files_matcher=matcher)
self.assertEqual(['file2', 'file3'], files)
def __init__(self, file_dir, file_type=None, file_extensions=None) -> None:
self._file_dir = file_dir
try:
self._values = list_files(file_dir, file_type, file_extensions)
except InvalidFileException as e:
LOGGER.warning('Failed to list files for ' + file_dir + ': ' + str(e))
self._values = []
def test_file_type_dir(self):
files = ['file1', 'file2']
test_utils.create_files(files)
test_utils.create_dir('my_dir')
actual_files = model_helper.list_files(test_utils.temp_folder,
file_type=FILE_TYPE_DIR)
self.assertEqual(['my_dir'], actual_files)
def __init__(self, file_dir, file_type=None, file_extensions=None) -> None:
self._file_dir = file_dir
try:
self._values = list_files(file_dir, file_type, file_extensions)
except InvalidFileException as e:
LOGGER.warning('Failed to list files for ' + file_dir + ': ' + str(e))
self._values = []
def test_file_extensions(self):
for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']:
for file in ['file1', 'file2']:
test_utils.create_file(file + '.' + extension)
test_utils.create_dir('my_dir' + '.' + extension)
files = model_helper.list_files(test_utils.temp_folder, file_extensions=['exe', 'pdf'])
self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'], files)
def test_plain_relative_path_in_subfolder(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files, 'sub')
subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
matcher = self.create_matcher([(os.path.join('sub', 'file2'))])
files = model_helper.list_files(subfolder_path,
excluded_files_matcher=matcher)
self.assertEqual(['file1', 'file3'], files)
def test_recursive_glob_relative_different_work_dir(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files, 'sub')
matcher = FileMatcher(['**'], test_utils.temp_folder + '2')
subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
files = model_helper.list_files(subfolder_path,
excluded_files_matcher=matcher)
self.assertEqual(['file1', 'file2', 'file3'], files)
def test_recursive_glob_relative_path_any_match(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files, 'sub')
matcher = self.create_matcher(['**'])
subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
files = model_helper.list_files(subfolder_path,
excluded_files_matcher=matcher)
self.assertEqual([], files)
def test_glob_absolute_path(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files)
matcher = self.create_matcher(
[file_utils.normalize_path('*1', test_utils.temp_folder)])
files = model_helper.list_files(test_utils.temp_folder,
excluded_files_matcher=matcher)
self.assertEqual(['file2', 'file3'], files)
def test_recursive_glob_relative_path_match_any_in_subfolder(self):
created_files = ['file1', 'file2', 'file3']
subfolder = os.path.join('a', 'b', 'c', 'd', 'e')
test_utils.create_files(created_files, subfolder)
matcher = self.create_matcher(['**/e/**'])
subfolder_path = os.path.join(test_utils.temp_folder, subfolder)
files = model_helper.list_files(subfolder_path,
excluded_files_matcher=matcher)
self.assertEqual([], files)
def test_multiple_files_non_recursive(self):
for dir in [None, 'documents', 'smth']:
for file in ['my.txt', 'file.dat']:
if dir:
test_utils.create_file(os.path.join(dir, dir + '_' + file))
else:
test_utils.create_file(file)
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_plain_absolute_path(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files)
excluded_file = os.path.abspath(
os.path.join(test_utils.temp_folder, 'file2'))
matcher = self.create_matcher([excluded_file])
files = model_helper.list_files(test_utils.temp_folder,
excluded_files_matcher=matcher)
self.assertEqual(['file1', 'file3'], files)
def test_multiple_files_non_recursive(self):
for dir in [None, 'documents', 'smth']:
for file in ['my.txt', 'file.dat']:
if dir:
test_utils.create_file(os.path.join(dir, dir + '_' + file))
else:
test_utils.create_file(file)
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'my.txt', 'smth'], files)
def test_recursive_glob_absolute_path(self):
created_files = ['file1', 'file2', 'file3']
test_utils.create_files(created_files, 'sub')
matcher = self.create_matcher(
[file_utils.normalize_path('**/file1', test_utils.temp_folder)])
subfolder_path = os.path.join(test_utils.temp_folder, 'sub')
files = model_helper.list_files(subfolder_path,
excluded_files_matcher=matcher)
self.assertEqual(['file2', 'file3'], files)
def test_file_extensions(self):
for extension in ['exe', 'dat', 'txt', 'sh', 'pdf', 'docx']:
for file in ['file1', 'file2']:
test_utils.create_file(file + '.' + extension)
test_utils.create_dir('my_dir' + '.' + extension)
files = model_helper.list_files(test_utils.temp_folder,
file_extensions=['exe', 'pdf'])
self.assertEqual(['file1.exe', 'file1.pdf', 'file2.exe', 'file2.pdf'],
files)
def test_multiple_exclusions_when_no_match(self):
created_files = ['fileA', 'fileB', 'fileC', 'fileD']
test_utils.create_files(created_files)
matcher = self.create_matcher([
'*2', 'file1',
file_utils.normalize_path('file4', test_utils.temp_folder)
])
files = model_helper.list_files(test_utils.temp_folder,
excluded_files_matcher=matcher)
self.assertEqual(created_files, files)
def test_recursive_glob_absolute_path_and_deep_nested(self):
created_files = ['file1', 'file2', 'file3']
abc_subfolder = os.path.join('a', 'b', 'c')
test_utils.create_files(created_files, abc_subfolder)
matcher = self.create_matcher(
[file_utils.normalize_path('**/file1', test_utils.temp_folder)])
abc_path = os.path.join(test_utils.temp_folder, abc_subfolder)
files = model_helper.list_files(abc_path,
excluded_files_matcher=matcher)
self.assertEqual(['file2', 'file3'], files)
def test_multiple_files(self):
test_utils.create_files(['My.txt', 'file.dat', 'test.sh'])
test_utils.create_dir('documents')
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'My.txt', 'test.sh'], files)
def test_single_file(self):
test_utils.create_file('my.txt')
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['my.txt'], files)
def test_single_file(self):
test_utils.create_file('my.txt')
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['my.txt'], files)
def test_multiple_files(self):
test_utils.create_files(['My.txt', 'file.dat', 'test.sh'])
test_utils.create_dir('documents')
files = model_helper.list_files(test_utils.temp_folder)
self.assertEqual(['documents', 'file.dat', 'My.txt', 'test.sh'], files)
