def _match(pattern, limit):
"""Find all matching paths to the pattern provided.
"""
files = glob.glob(pattern)
metadata = [
FileMetadata(f, os.path.getsize(f)) for f in files[:limit]
]
return MatchResult(pattern, metadata)
def _match(path_pattern, limit):
"""Find all matching paths to the pattern provided."""
file_infos = self._hdfs_client.ls(path_pattern,
detail=True)[:limit]
metadata_list = [
FileMetadata(file_info['name'], file_info['size'])
for file_info in file_infos
]
return MatchResult(path_pattern, metadata_list)
def _match(pattern, limit):
"""Find all matching paths to the pattern provided.
"""
if pattern.endswith('/'):
pattern += '*'
file_sizes = gcsio.GcsIO().size_of_files_in_glob(pattern, limit)
metadata_list = [
FileMetadata(path, size)
for path, size in file_sizes.iteritems()
]
return MatchResult(pattern, metadata_list)
def _match(path_pattern, limit):
"""Find all matching paths to the pattern provided."""
fs = self._hdfs_client.status(path_pattern, strict=False)
if fs and fs[_FILE_STATUS_TYPE] == _FILE_STATUS_TYPE_FILE:
file_statuses = [(fs[_FILE_STATUS_PATH_SUFFIX], fs)][:limit]
else:
file_statuses = self._hdfs_client.list(path_pattern,
status=True)[:limit]
metadata_list = [FileMetadata(file_status[1][_FILE_STATUS_NAME],
file_status[1][_FILE_STATUS_SIZE])
for file_status in file_statuses]
return MatchResult(path_pattern, metadata_list)
def _match(path_pattern, limit):
"""Find all matching paths to the pattern provided."""
fs = self._hdfs_client.status(path_pattern, strict=False)
if fs and fs[_FILE_STATUS_TYPE] == _FILE_STATUS_TYPE_FILE:
file_statuses = [(path_pattern, fs)][:limit]
else:
file_statuses = [(self._join(path_pattern, fs[0]), fs[1])
for fs in self._hdfs_client.list(path_pattern,
status=True)[:limit]]
metadata_list = [
FileMetadata(_HDFS_PREFIX + file_status[0],
file_status[1][_FILE_STATUS_LENGTH])
for file_status in file_statuses]
return MatchResult(path_pattern, metadata_list)
def match(self, patterns, limits=None):
test_context = get_current_test_context()
file_content_map = test_context.file_content_map
all_files = list(file_content_map.keys())
if limits is None:
limits = [None] * len(patterns)
results = []
for pattern, limit in zip(patterns, limits):
files = all_files[:limit]
metadata = [
FileMetadata(f, len(file_content_map[f]))
for f in files
]
results.append(MatchResult(pattern, metadata))
return results
