def test_list_objects():
s = uuid.uuid4()
prefix = "test_list_objects/" + str(s)
for i in [0, 3, 7, 11]:
f = TSAccessS3("smdebugcodebuildtest", prefix + "/" + format(i, "02"))
f.write(b"a")
f.close()
req1 = ListRequest(Bucket="smdebugcodebuildtest", Prefix=prefix)
req2 = ListRequest(Bucket="smdebugcodebuildtest",
Prefix="test_list_objects/",
Delimiter="/")
req3 = ListRequest(Bucket="smdebugcodebuildtest",
Prefix=prefix,
StartAfter=prefix + "/0")
req4 = ListRequest(Bucket="smdebugcodebuildtest",
Prefix=prefix,
StartAfter=prefix + "/03")
req5 = ListRequest(Bucket="smdebugcodebuildtest", Prefix=prefix + "/0")
files = S3Handler.list_prefixes([req1, req2, req3, req4, req5])
# test StartAfter and delimiters
assert len(files[0]) == 4
assert prefix + "/" in files[1]
assert len(files[2]) == 4
assert len(files[3]) == 2
assert len(files[4]) == 3
S3Handler.delete_prefix(path="s3://smdebugcodebuildtest/" + prefix)
def file_exists(file_path):
s3, bucket_name, key_name = is_s3(file_path)
if s3:
try:
request = ListRequest(bucket_name, key_name)
file_available = S3Handler.list_prefixes([request])[0]
if len(file_available) > 0:
return True
else:
return False
except ClientError as ex:
status_code = ex.response["ResponseMetadata"]["HTTPStatusCode"]
logger.info(f"Client error occurred : {ex}")
if status_code.startswith("4"):
raise ex
else:
return False
else:
return os.path.exists(file_path)
def test_merge_timeline_s3_write():
bucket_name = "smdebug-testing"
key_name = f"outputs/smprofiler-timeline-merge-test-{uuid.uuid4()}"
location = "s3://{}/{}".format(bucket_name, key_name)
tracefolder = "./tests/profiler/resources/merge_traces"
combined_timeline = MergedTimeline(tracefolder, output_directory=location)
combined_timeline.merge_timeline(0,
time.time() * CONVERT_TO_MICROSECS,
unit=MergeUnit.TIME)
start_step, end_step = 2, 4
tracefolder = "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output"
combined_timeline = MergedTimeline(tracefolder, output_directory=location)
combined_timeline.merge_timeline(start_step, end_step, unit=MergeUnit.STEP)
request = ListRequest(bucket_name, key_name)
files = S3Handler.list_prefixes([request])
assert len(files) == 1
assert len(files[0]) == 2
def check_dir_exists(path):
from smdebug.core.access_layer.s3handler import S3Handler, ListRequest
s3, bucket_name, key_name = is_s3(path)
if s3:
try:
request = ListRequest(bucket_name, key_name)
folder = S3Handler.list_prefixes([request])[0]
if len(folder) > 0 and has_training_ended(folder[-1]):
raise RuntimeError("The path:{} already exists on s3. "
"Please provide a directory path that does "
"not already exist.".format(path))
except ClientError as ex:
if ex.response["Error"]["Code"] == "NoSuchBucket":
# then we do not need to raise any error
pass
else:
# do not know the error
raise ex
elif os.path.exists(path) and has_training_ended(path):
raise RuntimeError("The path:{} already exists on local disk. "
"Please provide a directory path that does "
"not already exist".format(path))
def read_tf_profiler_metadata_file(file_path):
if not is_valid_tfprof_tracefilename(file_path):
return "", "0", "0"
s3, bucket_name, key_name = is_s3(file_path)
if s3:
try:
folder_name = "/".join(key_name.split("/")[:-4])
request = ListRequest(bucket_name, folder_name)
file_available = S3Handler.list_prefixes([request])
if len(file_available) > 0:
metadata_filename = list(
filter(lambda x: ".metadata" in x, file_available[0]))
if len(metadata_filename) > 0:
metadata_filename = metadata_filename[0]
metadata_filename = metadata_filename.split("/")[-1]
node_id, start, end = str(metadata_filename).split("_")
return node_id, start, end.split(".")[0]
else:
return "", "0", "0"
else:
return "", "0", "0"
except ClientError as ex:
status_code = ex.response["ResponseMetadata"]["HTTPStatusCode"]
logger.info(f"Client error occurred : {ex}")
if status_code.startswith("4"):
raise ex
else:
return "", "0", "0"
else:
folder_name = "/".join(file_path.split("/")[:-4])
metadata_filename = list(Path(folder_name).rglob("*.metadata"))
if len(metadata_filename) > 0:
metadata_filename = metadata_filename[0].name
node_id, start, end = str(metadata_filename).split("_")
return node_id, start, end.split(".")[0]
else:
return "", "0", "0"
def _list_s3_prefixes(list_info):
files = S3Handler.list_prefixes(list_info)
if len(files) == 1:
files = files[0]
return files