def mirror_stream(path, io_type, logger=None, buffering=1):
ensure_file(path)
from_stream = sys.stderr if io_type == ComputeIOType.STDERR else sys.stdout
with tailf(path, io_type, logger):
with open(path, 'a+', buffering=buffering) as to_stream:
with redirect_stream(to_stream=to_stream, from_stream=from_stream):
yield
def _upload_from_local(self, run_id, key, io_type):
path = self.get_local_path(run_id, key, io_type)
ensure_file(path)
key = self._blob_key(run_id, key, io_type)
with open(path, 'rb') as data:
blob = self._container_client.get_blob_client(key)
blob.upload_blob(data)
def _upload_from_local(self, run_id, key, io_type):
path = self.get_local_path(run_id, key, io_type)
ensure_file(path)
if self._skip_empty_files and os.stat(path).st_size == 0:
return
key = self._bucket_key(run_id, key, io_type)
with open(path, "rb") as data:
self._s3_session.upload_fileobj(data, self._s3_bucket, key)
def mirror_stream_to_file(stream, filepath):
ensure_file(filepath)
with tail_to_stream(filepath, stream) as pids:
with redirect_to_file(stream, filepath):
yield pids
def _upload_from_local(self, run_id, key, io_type):
path = self.get_local_path(run_id, key, io_type)
ensure_file(path)
key = self._bucket_key(run_id, key, io_type)
with open(path, 'rb') as data:
self._s3_session.upload_fileobj(data, self._s3_bucket, key)
def _upload_from_local(self, run_id, key, io_type):
path = self.get_local_path(run_id, key, io_type)
ensure_file(path)
with open(path, "rb") as data:
self._bucket.blob(self._bucket_key(run_id, key, io_type)).upload_from_file(data)
def mirror_stream(stream, path, buffering=1):
ensure_file(path)
with tailf(path):
with open(path, 'a+', buffering=buffering) as to_stream:
with redirect_stream(to_stream=to_stream, from_stream=stream):
yield
