def test_delete_prefix():
s = uuid.uuid4()
prefix = "test_delete_prefix/" + str(s)
for i in range(3):
f = TSAccessS3("smdebugcodebuildtest", prefix + "/" + str(i))
f.write(b"a")
f.close()
S3Handler.delete_prefix(path="s3://smdebugcodebuildtest/" + prefix)
entries = S3Handler.list_prefix(ListRequest("smdebugcodebuildtest",
prefix))
assert len(entries) == 0
def load_python_profile_stats(self):
"""Load the stats in by creating the profile directory, downloading each stats directory from s3 to the
profile directory, parsing the metadata from each stats directory name and creating a StepPythonProfileStats
entry corresponding to the stats file in the stats directory.
For cProfile, the stats file name is `python_stats`.
For pyinstrument, the stats file name `python_stats.json`.
"""
python_profile_stats = []
self._set_up_profile_dir()
list_request = ListRequest(Bucket=self.bucket_name, Prefix=self.prefix)
s3_filepaths = S3Handler.list_prefix(list_request)
object_requests = [
ReadObjectRequest(
os.path.join("s3://", self.bucket_name, s3_filepath))
for s3_filepath in s3_filepaths
]
objects = S3Handler.get_objects(object_requests)
for full_s3_filepath, object_data in zip(s3_filepaths, objects):
if os.path.basename(full_s3_filepath) not in (
CPROFILE_STATS_FILENAME,
PYINSTRUMENT_JSON_FILENAME,
PYINSTRUMENT_HTML_FILENAME,
):
get_logger().info(
f"Unknown file {full_s3_filepath} found, skipping...")
continue
path_components = full_s3_filepath.split("/")
framework, profiler_name, node_id, stats_dir, stats_file = path_components[
-5:]
stats_dir_path = os.path.join(self.profile_dir, node_id, stats_dir)
os.makedirs(stats_dir_path, exist_ok=True)
stats_file_path = os.path.join(stats_dir_path, stats_file)
with open(stats_file_path, "wb") as f:
f.write(object_data)
python_profile_stats.append(
StepPythonProfileStats(framework, profiler_name, node_id,
stats_dir, stats_file_path))
python_profile_stats.sort(
key=lambda x: (x.start_time_since_epoch_in_micros, x.node_id)
) # sort each step's stats by the step number, then node ID.
return python_profile_stats
def _refresh_event_file_list_s3_mode(self, list_dir):
event_files = [
x for x in S3Handler.list_prefix(list_dir)
if re.search(self._get_event_file_regex(), x)
]
for event_file in event_files:
event_file_full_path = f"s3://{list_dir.bucket}/{event_file}"
timestamp = self._get_timestamp_from_filename(event_file_full_path)
if timestamp is None:
self.logger.debug(
f"Unable to find timestamp from event file name {event_file}."
)
continue
if timestamp in self._timestamp_to_filename:
if event_file_full_path not in self._timestamp_to_filename[
timestamp]:
self._timestamp_to_filename[timestamp].append(
event_file_full_path)
else:
self._timestamp_to_filename[timestamp] = [event_file_full_path]
for timestamp in self._timestamp_to_filename:
self._timestamp_to_filename[timestamp].sort()
self._update_start_after_prefix()