def mirror_stream(path, io_type, logger=None, buffering=1): ensure_file(path) from_stream = sys.stderr if io_type == ComputeIOType.STDERR else sys.stdout with tailf(path, io_type, logger): with open(path, 'a+', buffering=buffering) as to_stream: with redirect_stream(to_stream=to_stream, from_stream=from_stream): yield
def _upload_from_local(self, run_id, key, io_type): path = self.get_local_path(run_id, key, io_type) ensure_file(path) key = self._blob_key(run_id, key, io_type) with open(path, 'rb') as data: blob = self._container_client.get_blob_client(key) blob.upload_blob(data)
def _upload_from_local(self, run_id, key, io_type): path = self.get_local_path(run_id, key, io_type) ensure_file(path) if self._skip_empty_files and os.stat(path).st_size == 0: return key = self._bucket_key(run_id, key, io_type) with open(path, "rb") as data: self._s3_session.upload_fileobj(data, self._s3_bucket, key)
def mirror_stream_to_file(stream, filepath): ensure_file(filepath) with tail_to_stream(filepath, stream) as pids: with redirect_to_file(stream, filepath): yield pids
def _upload_from_local(self, run_id, key, io_type): path = self.get_local_path(run_id, key, io_type) ensure_file(path) key = self._bucket_key(run_id, key, io_type) with open(path, 'rb') as data: self._s3_session.upload_fileobj(data, self._s3_bucket, key)
def _upload_from_local(self, run_id, key, io_type): path = self.get_local_path(run_id, key, io_type) ensure_file(path) with open(path, "rb") as data: self._bucket.blob(self._bucket_key(run_id, key, io_type)).upload_from_file(data)
def mirror_stream(stream, path, buffering=1): ensure_file(path) with tailf(path): with open(path, 'a+', buffering=buffering) as to_stream: with redirect_stream(to_stream=to_stream, from_stream=stream): yield