def _get_collection_files(self) -> list:
collection_files, _ = list_s3_objects(
self.bucket_name,
get_path_to_collections(self.prefix_name),
start_after_key=None,
delimiter="",
)
return collection_files
def list_index_files(self, start_after_key=None):
index_files, last_index_token = list_s3_objects(
self.bucket_name,
IndexFileLocationUtils.get_index_path(self.prefix_name),
start_after_key,
)
return index_files, last_index_token
def test_parse_worker_name_from_collection_file():
path = "s3://smdebug-testing/resources/one-index-file"
_, bucket_name, key_name = is_s3(path)
collection_files, _ = list_s3_objects(bucket_name, get_path_to_collections(key_name))
assert len(collection_files) == 1
collection_file = collection_files[0]
worker_name = get_worker_name_from_collection_file(collection_file)
assert worker_name == "/job:worker/replica:0/task:1/device:GPU:0"
file_name = "/tmp/collections/000000000/job-worker_1_collections.json"
worker_name = get_worker_name_from_collection_file(file_name)
assert worker_name == "job-worker_1"
def list_event_files(self, start_after_key=None):
event_files, last_index_token = list_s3_objects(
self.bucket_name, get_path_to_events_directory(self.prefix_name), start_after_key
)
return event_files
