def _get_collection_files(self) -> list: collection_files, _ = list_s3_objects( self.bucket_name, get_path_to_collections(self.prefix_name), start_after_key=None, delimiter="", ) return collection_files
def list_index_files(self, start_after_key=None): index_files, last_index_token = list_s3_objects( self.bucket_name, IndexFileLocationUtils.get_index_path(self.prefix_name), start_after_key, ) return index_files, last_index_token
def test_parse_worker_name_from_collection_file(): path = "s3://smdebug-testing/resources/one-index-file" _, bucket_name, key_name = is_s3(path) collection_files, _ = list_s3_objects(bucket_name, get_path_to_collections(key_name)) assert len(collection_files) == 1 collection_file = collection_files[0] worker_name = get_worker_name_from_collection_file(collection_file) assert worker_name == "/job:worker/replica:0/task:1/device:GPU:0" file_name = "/tmp/collections/000000000/job-worker_1_collections.json" worker_name = get_worker_name_from_collection_file(file_name) assert worker_name == "job-worker_1"
def list_event_files(self, start_after_key=None): event_files, last_index_token = list_s3_objects( self.bucket_name, get_path_to_events_directory(self.prefix_name), start_after_key ) return event_files