def get_stream_content(stream: IOBase,
seek: Optional[int] = None) -> (Optional[str], str):
if seek is not None and stream.seekable():
stream.seek(seek)
content = stream.read(limits.STREAM_SIZE_LIMIT_BYTES + 1)
default_ext = "txt" if isinstance(content, str) else "bin"
if isinstance(content, str):
content = content.encode("utf-8")
if limits.stream_size_exceeds_limit(len(content)):
return None, default_ext
while True:
chunk = stream.read(1024 * 1024)
if chunk is None:
continue
elif not chunk:
break
else:
if not isinstance(content, BytesIO):
content = BytesIO(content)
content.seek(0, 2)
if isinstance(chunk, str):
chunk = chunk.encode("utf-8")
if limits.stream_size_exceeds_limit(content.tell() + len(chunk)):
return None, default_ext
content.write(chunk)
if isinstance(content, BytesIO):
content = content.getvalue()
return content, default_ext
def test_attributes_iobase(sio: io.IOBase) -> None:
"""Check basic attrs/descriptor functions related to being subclass of io.IOBase."""
assert sio is not None
assert isinstance(sio, io.IOBase)
assert isinstance(sio, io.RawIOBase)
assert not isinstance(sio, io.TextIOBase)
assert sio.readable() is True
assert sio.writable() is True
assert sio.seekable() is False
assert sio.isatty() is False
assert sio.closed() is False
def write_wav(f, channels, sample_width=SAMPLE_WIDTH, raw_samples=False, seekable=None):
stream = wav_samples(channels, sample_width, raw_samples)
channel_count = 1 if inspect.isgenerator(channels) else len(channels)
output_seekable = IOBase.seekable(f) if seekable is None else seekable
if not output_seekable:
# protect the non-seekable file, since Wave_write will call tell
f = NonSeekableFileProxy(f)
w = wave.open(f)
w.setparams((
channel_count,
sample_width,
FRAME_RATE,
0, # setting zero frames, should update automatically as more frames written
COMPRESSION_TYPE,
COMPRESSION_NAME
))
if not output_seekable:
if wave_module_patched():
# set nframes to make wave module write data size of 0xFFFFFFF
w.setnframes((0xFFFFFFFF - 36) / w.getnchannels() / w.getsampwidth())
logger.debug("Setting frames to: {0}, {1}".format((w.getnframes()), w._nframes))
else:
w.setnframes((0x7FFFFFFF - 36) / w.getnchannels() / w.getsampwidth())
logger.debug("Setting frames to: {0}, {1}".format((w.getnframes()), w._nframes))
for chunk in buffer(stream):
logger.debug("Writing %d bytes..." % len(chunk))
if output_seekable:
w.writeframes(chunk)
else:
# tell wave module not to update nframes header field
# if output stream not seekable, e.g. STDOUT to a pipe
w.writeframesraw(chunk)
w.close()
async def create(
cls, name: str, architecture: str, content: io.IOBase, *,
title: str="",
filetype: BootResourceFileType=BootResourceFileType.TGZ,
chunk_size=(1 << 22), progress_callback=None):
"""Create a `BootResource`.
Creates an uploaded boot resource with `content`. The `content` is
uploaded in chunks of `chunk_size`. `content` must be seekable as the
first pass through the `content` will calculate the size and sha256
value then the second pass will perform the actual upload.
:param name: Name of the boot resource. Must be in format 'os/release'.
:type name: `str`
:param architecture: Architecture of the boot resource. Must be in
format 'arch/subarch'.
:type architecture: `str`
:param content: Content of the boot resource.
:type content: `io.IOBase`
:param title: Title of the boot resource.
:type title: `str`
:param filetype: Type of file in content.
:type filetype: `str`
:param chunk_size: Size in bytes to upload to MAAS in chunks.
(Default is 4 MiB).
:type chunk_size: `int`
:param progress_callback: Called to inform the current progress of the
upload. One argument is passed with the progress as a precentage.
If the resource was already complete and no content
needed to be uploaded then this callback will never be called.
:type progress_callback: Callable
:returns: Create boot resource.
:rtype: `BootResource`.
"""
if '/' not in name:
raise ValueError(
"name must be in format os/release; missing '/'")
if '/' not in architecture:
raise ValueError(
"architecture must be in format arch/subarch; missing '/'")
if not content.readable():
raise ValueError("content must be readable")
elif not content.seekable():
raise ValueError("content must be seekable")
if chunk_size <= 0:
raise ValueError(
"chunk_size must be greater than 0, not %d" % chunk_size)
size, sha256 = calc_size_and_sha265(content, chunk_size)
resource = cls._object(await cls._handler.create(
name=name, architecture=architecture, title=title,
filetype=filetype.value, size=str(size), sha256=sha256))
newest_set = max(resource.sets, default=None)
assert newest_set is not None
resource_set = resource.sets[newest_set]
assert len(resource_set.files) == 1
rfile = list(resource_set.files.values())[0]
if rfile.complete:
# Already created and fully up-to-date.
return resource
else:
# Upload in chunks and reload boot resource.
await cls._upload_chunks(
rfile, content, chunk_size, progress_callback)
return cls._object.read(resource.id)
async def create(cls,
name: str,
architecture: str,
content: io.IOBase,
*,
title: str = "",
filetype: BootResourceFileType = BootResourceFileType.TGZ,
chunk_size=(1 << 22),
progress_callback=None):
"""Create a `BootResource`.
Creates an uploaded boot resource with `content`. The `content` is
uploaded in chunks of `chunk_size`. `content` must be seekable as the
first pass through the `content` will calculate the size and sha256
value then the second pass will perform the actual upload.
:param name: Name of the boot resource. Must be in format 'os/release'.
:type name: `str`
:param architecture: Architecture of the boot resource. Must be in
format 'arch/subarch'.
:type architecture: `str`
:param content: Content of the boot resource.
:type content: `io.IOBase`
:param title: Title of the boot resource.
:type title: `str`
:param filetype: Type of file in content.
:type filetype: `str`
:param chunk_size: Size in bytes to upload to MAAS in chunks.
(Default is 4 MiB).
:type chunk_size: `int`
:param progress_callback: Called to inform the current progress of the
upload. One argument is passed with the progress as a precentage.
If the resource was already complete and no content
needed to be uploaded then this callback will never be called.
:type progress_callback: Callable
:returns: Create boot resource.
:rtype: `BootResource`.
"""
if '/' not in name:
raise ValueError("name must be in format os/release; missing '/'")
if '/' not in architecture:
raise ValueError(
"architecture must be in format arch/subarch; missing '/'")
if not content.readable():
raise ValueError("content must be readable")
elif not content.seekable():
raise ValueError("content must be seekable")
if chunk_size <= 0:
raise ValueError("chunk_size must be greater than 0, not %d" %
chunk_size)
size, sha256 = calc_size_and_sha265(content, chunk_size)
resource = cls._object(await
cls._handler.create(name=name,
architecture=architecture,
title=title,
filetype=filetype.value,
size=str(size),
sha256=sha256))
newest_set = max(resource.sets, default=None)
assert newest_set is not None
resource_set = resource.sets[newest_set]
assert len(resource_set.files) == 1
rfile = list(resource_set.files.values())[0]
if rfile.complete:
# Already created and fully up-to-date.
return resource
else:
# Upload in chunks and reload boot resource.
await cls._upload_chunks(rfile, content, chunk_size,
progress_callback)
return cls._object.read(resource.id)
def __init__(self, raw: io.IOBase):
assert raw.seekable()
self.raw = raw
assert self.check_is_block_gzip()
