Exemplos de TSAccessS3 em Python

Exemplo n.º 1

Arquivo: test_training_end.py Projeto: vandanavk/sagemaker-debugger

def test_s3_training_end():
    s3dir = "s3://smdebugcodebuildtest/training_end_test_dir"
    _, bucket, key = is_s3(s3dir)
    f = TSAccessS3(bucket_name=bucket, key_name=key)
    f.close()
    training_has_ended(s3dir)
    assert has_training_ended(s3dir) is True
    delete_s3_prefixes(bucket, key)

Exemplo n.º 2

def test_s3_training_end():
    s3key = str(uuid.uuid4())
    s3dir = f"s3://smdebugcodebuildtest/ok_to_delete_{s3key}"
    _, bucket, key = is_s3(s3dir)
    f = TSAccessS3(bucket_name=bucket, key_name=key)
    f.close()
    training_has_ended(s3dir)
    assert has_training_ended(s3dir) is True
    delete_s3_prefixes(bucket, key)

Exemplo n.º 3

 def __init__(self, path, write_checksum):
     self.write_checksum = write_checksum
     s3, bucket_name, key_name = is_s3(path)
     try:
         if s3:
             self._writer = TSAccessS3(bucket_name, key_name)
         else:
             self._writer = TSAccessFile(path, "wb")
     except (OSError, IOError) as err:
         raise ValueError("failed to open {}: {}".format(path, str(err)))

Exemplo n.º 4

Arquivo: record_reader.py Projeto: vandanavk/sagemaker-debugger

 def __init__(self, path):
     s3, bucket_name, key_name = is_s3(path)
     try:
         if s3:
             self._reader = TSAccessS3(bucket_name, key_name)
         else:
             self._reader = TSAccessFile(path, "rb")
     except (OSError, IOError) as err:
         raise ValueError("failed to open {}: {}".format(path, str(err)))
     except:
         raise
     self._reader.ingest_all()

Exemplo n.º 5

def write_tf_profiler_metadata_file(file_path):
    if not file_path.endswith(".metadata"):
        return
    s3, bucket_name, key_name = is_s3(file_path)
    if s3:
        writer = TSAccessS3(bucket_name, key_name, binary=False)
    else:
        writer = TSAccessFile(file_path, "a+")
    writer.flush()
    try:
        writer.close()
    except OSError:
        """

Exemplo n.º 6

 def open(self, file_path):
     """
     Open the trace event file
     """
     s3, bucket_name, key_name = is_s3(file_path)
     try:
         if s3:
             self._writer = TSAccessS3(bucket_name, key_name, binary=False)
         else:
             self._writer = TSAccessFile(file_path, "w")
     except (OSError, IOError) as err:
         logger.debug(
             f"Sagemaker-Debugger: failed to open {file_path}: {str(err)}")
     start, length = self._writer.write("[\n")
     self.bytes_written += length

Exemplo n.º 7

Arquivo: index_file_writer.py Projeto: vandanavk/sagemaker-debugger

 def _init_writer(self):
     s3, bucket_name, key_name = is_s3(self.file_path)
     if s3:
         self.writer = TSAccessS3(bucket_name, key_name, binary=False)
     else:
         self.writer = TSAccessFile(self.file_path, "a+")