def write_from_stream(self, stream: io.IOBase, *, append=True, atomic=False):
if not isinstance(stream, io.IOBase):
raise TypeError(type(stream))
if not stream.readable():
raise ValueError('stream is unable to read.')
mode = 'a' if append else 'w'
if not isinstance(stream, io.TextIOBase):
mode += 'b'
return self.write(stream, mode=mode, atomic=atomic)
def test_attributes_iobase(sio: io.IOBase) -> None:
"""Check basic attrs/descriptor functions related to being subclass of io.IOBase."""
assert sio is not None
assert isinstance(sio, io.IOBase)
assert isinstance(sio, io.RawIOBase)
assert not isinstance(sio, io.TextIOBase)
assert sio.readable() is True
assert sio.writable() is True
assert sio.seekable() is False
assert sio.isatty() is False
assert sio.closed() is False
def __init__(self, handle : io.IOBase, key : Union[Iterable[int], int, str, None]):
self._init_key(key)
self._handle = handle
self._size = 0
self._pos = 0
self._buffer = bytearray(8)
if handle.readable():
# Don't assert due to BriceIsSmart
handle.read(12)
self._size = self._readu32()
if handle.writable():
self._write_header()
def get_md5_from_stream(src: io.IOBase) -> str:
"""calculate md5 of src stream. The stream could been
from a file(mode='rb')/network-stream/stringio or any other readable
object in BINARY stream. This method will NOT close the stream!
Return the MD5 hex digest number."""
if not isinstance(src, io.IOBase) or not src.readable():
raise Exception("src is not stream or unreadable")
m: hashlib._hashlib.HASH = hashlib.md5()
while True:
b = src.read(4096)
if not b:
break
m.update(b)
res = m.hexdigest()
return res
def __get_segs_from_stm(self, stm: io.IOBase) -> iter:
""""""
if stm is None or not stm.readable():
raise Exception("Data stream is unreadable")
segindex = 0
segline = 1 # BCP一行就是一个数据段
for line in stm:
try:
items: list = line.split('\t')
if not isinstance(items, list) or len(items) < 1:
continue
# bcp字段
seg: DataSeg = DataSeg()
self._append_item(seg, 'ispid', items, 0)
self._append_item(seg, 'sip', items, 1)
self._append_item(seg, 'dip', items, 2)
self._append_item(seg, 'sport', items, 3)
self._append_item(seg, 'dport', items, 4)
self._append_item(seg, 'time', items, 5, int)
self._append_item(seg, 'phone', items, 6)
self._append_item(seg, 'imsi', items, 7)
self._append_item(seg, 'equipmentid', items, 8)
self._append_item(seg, 'longitude', items, 9)
self._append_item(seg, 'latitude', items, 10)
self._append_item(seg, 'maintype', items, 11)
self._append_item(seg, 'osversion', items, 12)
self._append_item(seg, 'dataid', items, 13)
self._append_item(seg, 'msisdn', items, 14)
self._append_item(seg, 'lac', items, 15)
self._append_item(seg, 'ci', items, 16)
self._append_item(seg, 'url', items, 17)
self._append_item(seg, 'host', items, 18)
self._append_item(seg, 'cookie', items, 19)
seg.segindex = segindex
seg.segline = segline
segline += 1
segindex += 1
yield seg
except Exception as ex:
self._logger.error(
"Parse one line in bcp file error: {}".format(ex))
def text_to_node(self, iio: io.IOBase, parts_list):
'''
CNLを読み込み
'''
def find_part(name, path):
if name != None:
for y in parts_list:
if y.name == name:
return y
elif path != None:
for y in parts_list:
if y.path == path:
return y
index = 0
while iio.readable():
line=iio.readline().strip()
line = line.split(' ')
if line[0] == 'None':
index+=1
elif line[0] == '[Name]':
name = line[1]
elif line[0] == '[Path]':
if len(line) == 1:
path = ''
else:
path = line[1]
elif line[0] == '[Child]':
self.children[index].connect(find_part(name, path))
text_to_node(self.children[index], iio, parts_list)
index+=1
elif line[0] == '[Parent]':
return
elif line[0] == 'MATERIAL':
return
async def create(
cls, name: str, architecture: str, content: io.IOBase, *,
title: str="",
filetype: BootResourceFileType=BootResourceFileType.TGZ,
chunk_size=(1 << 22), progress_callback=None):
"""Create a `BootResource`.
Creates an uploaded boot resource with `content`. The `content` is
uploaded in chunks of `chunk_size`. `content` must be seekable as the
first pass through the `content` will calculate the size and sha256
value then the second pass will perform the actual upload.
:param name: Name of the boot resource. Must be in format 'os/release'.
:type name: `str`
:param architecture: Architecture of the boot resource. Must be in
format 'arch/subarch'.
:type architecture: `str`
:param content: Content of the boot resource.
:type content: `io.IOBase`
:param title: Title of the boot resource.
:type title: `str`
:param filetype: Type of file in content.
:type filetype: `str`
:param chunk_size: Size in bytes to upload to MAAS in chunks.
(Default is 4 MiB).
:type chunk_size: `int`
:param progress_callback: Called to inform the current progress of the
upload. One argument is passed with the progress as a precentage.
If the resource was already complete and no content
needed to be uploaded then this callback will never be called.
:type progress_callback: Callable
:returns: Create boot resource.
:rtype: `BootResource`.
"""
if '/' not in name:
raise ValueError(
"name must be in format os/release; missing '/'")
if '/' not in architecture:
raise ValueError(
"architecture must be in format arch/subarch; missing '/'")
if not content.readable():
raise ValueError("content must be readable")
elif not content.seekable():
raise ValueError("content must be seekable")
if chunk_size <= 0:
raise ValueError(
"chunk_size must be greater than 0, not %d" % chunk_size)
size, sha256 = calc_size_and_sha265(content, chunk_size)
resource = cls._object(await cls._handler.create(
name=name, architecture=architecture, title=title,
filetype=filetype.value, size=str(size), sha256=sha256))
newest_set = max(resource.sets, default=None)
assert newest_set is not None
resource_set = resource.sets[newest_set]
assert len(resource_set.files) == 1
rfile = list(resource_set.files.values())[0]
if rfile.complete:
# Already created and fully up-to-date.
return resource
else:
# Upload in chunks and reload boot resource.
await cls._upload_chunks(
rfile, content, chunk_size, progress_callback)
return cls._object.read(resource.id)
async def create(cls,
name: str,
architecture: str,
content: io.IOBase,
*,
title: str = "",
filetype: BootResourceFileType = BootResourceFileType.TGZ,
chunk_size=(1 << 22),
progress_callback=None):
"""Create a `BootResource`.
Creates an uploaded boot resource with `content`. The `content` is
uploaded in chunks of `chunk_size`. `content` must be seekable as the
first pass through the `content` will calculate the size and sha256
value then the second pass will perform the actual upload.
:param name: Name of the boot resource. Must be in format 'os/release'.
:type name: `str`
:param architecture: Architecture of the boot resource. Must be in
format 'arch/subarch'.
:type architecture: `str`
:param content: Content of the boot resource.
:type content: `io.IOBase`
:param title: Title of the boot resource.
:type title: `str`
:param filetype: Type of file in content.
:type filetype: `str`
:param chunk_size: Size in bytes to upload to MAAS in chunks.
(Default is 4 MiB).
:type chunk_size: `int`
:param progress_callback: Called to inform the current progress of the
upload. One argument is passed with the progress as a precentage.
If the resource was already complete and no content
needed to be uploaded then this callback will never be called.
:type progress_callback: Callable
:returns: Create boot resource.
:rtype: `BootResource`.
"""
if '/' not in name:
raise ValueError("name must be in format os/release; missing '/'")
if '/' not in architecture:
raise ValueError(
"architecture must be in format arch/subarch; missing '/'")
if not content.readable():
raise ValueError("content must be readable")
elif not content.seekable():
raise ValueError("content must be seekable")
if chunk_size <= 0:
raise ValueError("chunk_size must be greater than 0, not %d" %
chunk_size)
size, sha256 = calc_size_and_sha265(content, chunk_size)
resource = cls._object(await
cls._handler.create(name=name,
architecture=architecture,
title=title,
filetype=filetype.value,
size=str(size),
sha256=sha256))
newest_set = max(resource.sets, default=None)
assert newest_set is not None
resource_set = resource.sets[newest_set]
assert len(resource_set.files) == 1
rfile = list(resource_set.files.values())[0]
if rfile.complete:
# Already created and fully up-to-date.
return resource
else:
# Upload in chunks and reload boot resource.
await cls._upload_chunks(rfile, content, chunk_size,
progress_callback)
return cls._object.read(resource.id)
def get_sha512_from_stream(src: io.IOBase) -> str:
"""get the sha512 of the src bytes, return hex str result"""
if not isinstance(src, io.IOBase) or not src.readable():
raise Exception("src is not stream or unreadable")
m = hashlib.sha512()
return calc_hash(src, m)
def loadf(self, fp: IOBase, options: dict) -> Any:
'load a obj from a file-like object.'
assert fp.readable()
return self.load(fp.read(), options)
