def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args,
**kwargs):
rounded_sampling_rate = int(round((self.sampling_rate * 4096) / 44100))
data_out.write(
self.struct.pack(kwargs['end_section_offset'],
rounded_sampling_rate, self.flags.value, self.uk1,
self.size))
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = util.logging.get_logger("project-receiver")
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
num_bytes = 0
while True:
data = sock.recv(util.MAX_PACKET)
if not data:
break
logger.info("Received %d bytes", len(data))
dest.write(data)
num_bytes += len(data)
dest.flush()
return num_bytes
def write(self, io_out: BufferedIOBase, v: Optional[Dict[str, Any]], otherfields: Dict[str, Any]) -> None:
# If they didn't specify this tlvstream, it's empty.
if v is None:
return
# Make a tuple of (fieldnum, val_to_bin, val) so we can sort into
# ascending order as TLV spec requires.
def write_raw_val(iobuf: BufferedIOBase, val: Any, otherfields: Dict[str, Any]) -> None:
iobuf.write(val)
def get_value(tup):
"""Get value from num, fun, val tuple"""
return tup[0]
ordered: List[Tuple[int,
Callable[[BufferedIOBase, Any, Dict[str, Any]], None],
Any]] = []
for fieldname in v:
f = self.find_field(fieldname)
if f is None:
# fieldname can be an integer for a raw field.
ordered.append((int(fieldname), write_raw_val, v[fieldname]))
else:
ordered.append((f.number, f.write, v[fieldname]))
ordered.sort(key=get_value)
for typenum, writefunc, val in ordered:
buf = BytesIO()
writefunc(cast(BufferedIOBase, buf), val, otherfields)
BigSizeType.write(io_out, typenum)
BigSizeType.write(io_out, len(buf.getvalue()))
io_out.write(buf.getvalue())
def __init__(self, bucket, key, *, chunk_size=DEFAULT_CHUNK_SIZE,
max_file_log_time=DEFAULT_ROTATION_TIME_SECS, max_file_size_bytes=MAX_FILE_SIZE_BYTES,
encoder='utf-8', workers=2, compress=False, key_id=None, secret=None, token=None):
self.session = Session(aws_access_key_id=key_id, aws_secret_access_key=secret, aws_session_token=token)
self.s3 = self.session.resource('s3')
self.start_time = int(time.time())
self.key = key.strip('/')
self.chunk_size = chunk_size
self.max_file_log_time = max_file_log_time
self.max_file_size_bytes = max_file_size_bytes
self.current_file_name = "{}_{}".format(key, int(time.time()))
if compress:
self.current_file_name = "{}.gz".format(self.current_file_name)
self.encoder = encoder
try:
self.s3.meta.client.head_bucket(Bucket=bucket)
except Exception:
raise ValueError('Bucket %s does not exist, or missing permissions' % bucket)
self._bucket = self.s3.Bucket(bucket)
self._current_object = self._get_stream_object(self.current_file_name)
self.workers = [threading.Thread(target=task_worker, args=(self._rotation_queue,)).start() for _ in
range(int(max(workers, MIN_WORKERS_NUM) / 2) + 1)]
self.stream_bg_workers = [threading.Thread(target=task_worker, args=(self._stream_buffer_queue,)).start() for _
in range(max(int(max(workers, MIN_WORKERS_NUM) / 2), 1))]
self._is_open = True
self.compress = compress
BufferedIOBase.__init__(self)
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("hw5-receiver")
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
num_bytes = 0
sequenceNumber = 0
while True:
data = sock.recv(homework5.MAX_PACKET)
if not data:
break
header = data[:4]
data = data[4:]
tempNumber = struct.unpack("i", header)[0]
if data[4:] is b'':
break
logger.info("Received %d bytes", len(data))
if tempNumber > sequenceNumber:
sequenceNumber = tempNumber
dest.write(data)
num_bytes += len(data)
dest.flush()
sock.send(struct.pack("i", sequenceNumber))
return num_bytes
def __init__(self, file_obj: BufferedIOBase,
key_map: KeyMap,
init=False, *,
portion=3/4):
self.file = file_obj
self.maps = [key_map]
self._portion = portion
# set the first and last pointers
file_obj.seek(0)
if init:
# empty pointers
self._first = None
self._last = None
# start fill tracking
self._filled = 0
else:
# read pointers
self._first = int.from_bytes(file_obj.read(32), 'little')
self._last = int.from_bytes(file_obj.read(32), 'little')
# get current fill
self._filled = int.from_bytes(file_obj.read(32), 'little')
# add offset for pointers
self._pos = 64
# initialize key cache
self._keys = {}
# track if currently expanding db
self._expansion = None
# set up iterable variable
self._current = None
def __init__(self, key_id, secret, stream_name, region, partition, chunk_size=DEFAULT_CHUNK_SIZE, encoder='utf-8',
workers=2):
self.kinesis = client('kinesis', region_name=region, aws_access_key_id=key_id,
aws_secret_access_key=secret)
self.chunk_size = chunk_size
self.stream_name = stream_name
self.region = region
self.tasks = queue.Queue()
self.partition = partition
self.encoder = encoder
try:
stream_desc = self.kinesis.describe_stream(StreamName=self.stream_name)
if stream_desc['StreamDescription']['StreamStatus'] != 'ACTIVE':
raise AssertionError
except Exception:
raise ValueError('Kinesis stream %s does not exist or inactive, or insufficient permissions' % stream_name)
self.workers = [threading.Thread(target=task_worker, args=(self.tasks,)).start() for _ in
range(int(max(workers, MIN_WORKERS_NUM) / 2) + 1)]
self._stream = BytesIO()
self._is_open = True
BufferedIOBase.__init__(self)
def unpack(
self,
buffer: io.BufferedIOBase,
unicode: bool = None,
version: Version = None,
gx_lists: list[GXList] = None,
gx_list_indices: dict[int, int] = None,
):
if any(var is None
for var in (unicode, version, gx_lists, gx_list_indices)):
raise ValueError(
"Not all required keywords were passed to `Material.unpack()`."
)
data = self.STRUCT.unpack(buffer)
encoding = "utf-16-le" if unicode else "shift_jis_2004"
self.name = read_chars_from_buffer(buffer,
offset=data.pop("__name_offset"),
encoding=encoding)
self.mtd_path = read_chars_from_buffer(
buffer, offset=data.pop("__mtd_path_offset"), encoding=encoding)
gx_offset = data.pop("__gx_offset")
if gx_offset == 0:
self.gx_index = -1
elif gx_offset in gx_list_indices:
self.gx_index = gx_list_indices[gx_offset]
else:
gx_list_indices[gx_offset] = len(gx_lists)
material_offset = buffer.tell()
buffer.seek(gx_offset)
gx_lists.append(GXList(buffer, version))
buffer.seek(material_offset)
self.set(**data)
def read_socket_to_file(bufferedReader: BufferedReader, file: BufferedIOBase,
size):
"""
Reads data from a socket to a file, logging progress if data is big.
"""
logProgress = False
if size > big_file_threshold:
logProgress = True
try:
pointer = 0
while pointer < size:
data = bufferedReader.read(min(buffer_size, size - pointer))
file.write(data)
pointer += len(data)
if logProgress:
print("Receiving file",
pointer,
"/",
size,
"bytes received",
end='\r')
if logProgress: print()
except Exception:
print("Error while receiving file.")
def __init__(self,
name,
mode='r',
buffer_size=None,
max_buffers=0,
max_workers=None,
**kwargs):
if 'a' in mode:
# TODO: Implement append mode and remove this exception
raise NotImplementedError('Not implemented yet in Pycosio')
BufferedIOBase.__init__(self)
ObjectIOBase.__init__(self, name, mode=mode)
WorkerPoolBase.__init__(self, max_workers)
# Instantiate raw IO
self._raw = self._RAW_CLASS(name, mode=mode, **kwargs)
self._raw._is_raw_of_buffered = True
# Link to RAW methods
self._mode = self._raw.mode
self._name = self._raw.name
self._client_kwargs = self._raw._client_kwargs
# Initializes buffer
if not buffer_size or buffer_size < 0:
self._buffer_size = self.DEFAULT_BUFFER_SIZE
elif buffer_size < self.MINIMUM_BUFFER_SIZE:
self._buffer_size = self.MINIMUM_BUFFER_SIZE
elif (self.MAXIMUM_BUFFER_SIZE
and buffer_size > self.MAXIMUM_BUFFER_SIZE):
self._buffer_size = self.MAXIMUM_BUFFER_SIZE
else:
self._buffer_size = buffer_size
# Initialize write mode
if self._writable:
self._max_buffers = max_buffers
self._buffer_seek = 0
self._write_buffer = bytearray(self._buffer_size)
self._seekable = False
self._write_futures = []
self._raw_flush = self._raw._flush
# Size used only with random write access
# Value will be lazy evaluated latter if needed.
self._size_synched = False
self._size = 0
self._size_lock = Lock()
# Initialize read mode
else:
self._size = self._raw._size
self._read_range = self.raw._read_range
if max_buffers:
self._max_buffers = max_buffers
else:
self._max_buffers = ceil(self._size / self._buffer_size)
self._read_queue = dict()
def write(self,
stream: BufferedIOBase,
nodes: List[SceneNode],
mode=MeshWriter.OutputMode.BinaryMode) -> bool:
Logger.log("i", "starting ChituCodeWriter.")
if mode != MeshWriter.OutputMode.TextMode:
Logger.log("e", "ChituCodeWriter does not support non-text mode.")
self.setInformation(
catalog.i18nc(
"@error:not supported",
"ChituCodeWriter does not support non-text mode."))
return False
gcode_textio = StringIO() #We have to convert the g-code into bytes.
gcode_writer = cast(
MeshWriter,
PluginRegistry.getInstance().getPluginObject("GCodeWriter"))
success = gcode_writer.write(gcode_textio, None)
if not success:
self.setInformation(gcode_writer.getInformation())
return False
result = self.modify(gcode_textio.getvalue())
stream.write(result)
Logger.log("i", "ChituWriter done")
return True
def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args,
**kwargs):
rounded_sampling_rate = int(round((self.sampling_rate * 4096) / 48000))
data_out.write(
self.struct.pack(self.sampling_rate, rounded_sampling_rate,
self.volume_level, self.flags.value, self.uk1,
self.uk2, self.size))
def parse(cls, data_in: BufferedIOBase, conf: Configuration, *args,
**kwargs):
data_in.seek(4, SEEK_CUR) # Group header offset
n_sound_effects = int.from_bytes(data_in.read(4), 'little')
data_in.seek(
8, SEEK_CUR
) # End offset (4 bytes) | Sum of group VAGs' sizes (4 bytes)
return cls(n_sound_effects=n_sound_effects)
def readtimes(f: io.BufferedIOBase) -> List[float]:
b = f.read(4)
numof_times = struct.unpack_from("<L", b, 0)[0]
times = [0.0 for _ in range(numof_times)]
bs = f.read(8 * numof_times)
for i, _ in enumerate(times):
times[i] = struct.unpack_from("<d", bs, 8 * i)
return times
def uncompress(self, f_in: BufferedIOBase, f_out: BufferedIOBase):
while True:
start = uncompress_num(f_in)
if start is None:
break
byte_len = uncompress_num(f_in)
# print(hex(start), hex(byte_len))
f_out.write(get_bytes(self.dict, start, byte_len))
def write(self, io_out: BufferedIOBase, v: int, otherfields: Dict[str, Any]) -> None:
binval = struct.pack('>Q', v)
while len(binval) != 0 and binval[0] == 0:
binval = binval[1:]
if len(binval) > self.maxbytes:
raise ValueError('{} exceeds maximum {} capacity'
.format(v, self.name))
io_out.write(binval)
def parse_message_pairs(stream: BufferedIOBase):
parser = intialize_parser(parse_message_pair)
data = stream.read(1024)
while data:
for rr in parse(parser, data):
yield rr
data = stream.read(1024)
def download_file_from_output_directory(base_url: str, relative_path: str,
output: BufferedIOBase):
url = (base_url if base_url.endswith('/') else base_url +
'/') + relative_path.replace('\\', '/').replace('//', '/')
resp = requests.get(url, stream=True)
for chunk in resp.iter_content(1024):
if chunk:
output.write(chunk)
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
global EXPECTED_SEQUENCE
logger = homework5.logging.get_logger("hw5-receiver")
num_bytes = 0
while True:
try:
data = sock.recv(homework5.MAX_PACKET)
# Kill the process as soon as there is nothing else to send
if not data:
break
# Gather the packet and retrieve the sequence number and data
new_packet = decode_packet(data)
header_only = new_packet[0]
data_only = new_packet[1]
# Check if the packet received is not off
if header_only == EXPECTED_SEQUENCE:
# If the packet received also contains data, then send an ack
if data_only is not None:
# Send an Acknowledgement that the data received corresponds
# to expected value
sock.send(make_packet(EXPECTED_SEQUENCE))
logger.info("Received %d bytes", len(data_only))
dest.write(data_only)
num_bytes += len(data_only)
dest.flush()
# Update the expected sequence if the data that we received
# is the one that was expected
update_sequence()
# If the packet sequence is off, resent the
else:
sock.send(make_packet(EXPECTED_SEQUENCE))
# If there was a timeout, continue
except socket.timeout:
continue
return num_bytes
def _read_data_frame(rfile: BufferedIOBase):
frame = {}
net_bytes = ord(rfile.read(1))
frame["FIN"] = net_bytes >> 7
frame["RSV1"] = (net_bytes & 0x40) >> 6
frame["RSV2"] = (net_bytes & 0x20) >> 5
frame["RSV3"] = (net_bytes & 0x10) >> 4
frame["opcode"] = net_bytes & 0x0F
if frame["RSV1"] != 0 or frame["RSV2"] != 0 or frame["RSV3"] != 0:
raise FrameError(
"Unsupported feature. RSV1, RSV2 or RSV3 has a non-zero value.",
frame,
)
if not frame["opcode"] in OPCODE.values():
raise FrameError("Unsupported opcode value.", frame)
if frame["FIN"] == 0 and frame["opcode"] != OPCODE["continueation"]:
raise FrameError(
"FIN bit not set for a non-continueation frame.", frame
)
if frame["opcode"] in CONTROL_OPCODES and frame["FIN"] == 0:
raise FrameError("FIN bit not set for a control frame.", frame)
net_bytes = ord(rfile.read(1))
mask_bit = net_bytes >> 7
if mask_bit == 0:
raise FrameError("Unmasked frame from client.", frame)
length1 = net_bytes & 0x7F
if frame["opcode"] in CONTROL_OPCODES and length1 > 125:
raise FrameError("Control frame with invalid payload length.", frame)
try:
length = _read_payload_length(length1, rfile)
except InvalidLengthError as error:
raise FrameError(
f"Invalid payload length of {error.length} bytes.", frame
)
masking_key = rfile.read(4)
encoded_payload = rfile.read(length)
frame["payload"] = _decode_payload(masking_key, encoded_payload)
if frame["opcode"] == OPCODE["close"] and frame["payload"]:
frame["status_code"] = int.from_bytes(
frame["payload"][0:2], byteorder="big"
)
if length > 2:
# /!\ may raise UnicodeError /!\
frame["close_reason"] = frame["payload"][2:].decode()
return frame
def _read_payload_length(payload_length1: int, rfile: BufferedIOBase):
final_length = payload_length1
if payload_length1 == 126:
final_length = int.from_bytes(rfile.read(2), byteorder="big")
elif payload_length1 == 127:
final_length = int.from_bytes(rfile.read(8), byteorder="big")
if final_length >> 63 == 1:
raise InvalidLengthError(final_length)
return final_length
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("hw5-receiver")
num_bytes = 0
sequence = 0
count = 0
while True:
data = sock.recv(homework5.MAX_PACKET)
#print("\nRECEIVER DATA[0]->", data[:2])
if data[0] == sequence:
sock.send(bytes([sequence]))
#print('\n')
print("RECEVIER SENT THESE BYTES->", str(data[0]))
buff = data[1:]
logger.info("Received %d bytes", len(buff))
#print('\n')
dest.write(buff)
count = 0
num_bytes += len(buff)
dest.flush()
if sequence == 255:
sequence = 0
else:
sequence+=1
continue
elif data[:2] == FIN:
sock.send(FIN)
# print("RECV CLOSED SUCCESSFULLY")
sock.close()
break
if data[0] != sequence:
# sequence-=1
# if count ==0:
sock.send(bytes([data[0]]))
print("WRONG SEQUENCE -- RECEVIER GOT THESE BYTES->", str(data[0]))
print("looking for these bytes->",sequence)
# count+=1
# sock.send(bytes([data[0]]))
# print("DATA was dropped I am RECIEVER and still looking for sequence",str(bytes([sequence])))
continue
break
return num_bytes
def read_object(cls, file: BufferedIOBase):
new_key = file.read(cls.key_size)
obj_size = file.read(cls.key_size)
obj_size = int.from_bytes(obj_size, 'little')
new_data = file.read(obj_size)
return cls(new_key, new_data)
def compress_num(nums: list, f_out: BufferedIOBase):
# print(hex(nums[0]), hex(nums[1]))
# use most significant bit to tell the end of a number
buf = bytearray()
for num in nums:
while num >= 128:
buf.append(num & 0x7F)
num >>= 7
else:
buf.append(num + 128)
f_out.write(buf)
def __fill_buf(self, f_in: BufferedIOBase) -> bool:
next_bytes = f_in.read(self.read_len)
if len(next_bytes) == 0:
return False
cur_process = f_in.tell() * 100 // self.length
if cur_process != self.process:
self.process = cur_process
print(str(cur_process) + "%")
self.buf.extend(next_bytes)
return True
def transfer_destination_io(self,
*,
source: str,
destination: io.BufferedIOBase,
chunk_size: int = 4096) -> None:
"""Transfer from source file to destination IO.
Note that this can't use std{in,out}=open(...) due to LP #1849753.
:param source: The source path, prefixed with <name:> for a path inside
the instance.
:param destination: An IO stream to write to.
:raises MultipassError: On error.
"""
command = [str(self.multipass_path), "transfer", source, "-"]
proc = subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
)
# Should never happen, but pyright makes noise.
assert proc.stdout is not None
while True:
written = proc.stdout.read(chunk_size)
if written:
destination.write(written)
if len(written) < chunk_size:
logger.debug("Finished streaming standard output")
break
while True:
try:
out, _ = proc.communicate(timeout=1)
except subprocess.TimeoutExpired:
continue
if out:
destination.write(out)
if proc.returncode == 0:
logger.debug("Process completed")
break
if proc.returncode is not None:
raise MultipassError(
command=command,
returncode=proc.returncode,
msg=f"Failed to transfer file {source!r} to destination.",
)
def __init__(self, out=None, encoding='iso-8859-1', short_empty_elements=False):
xmltodict.XMLGenerator.__bases__.__init__(self)
buf = BufferedIOBase()
buf.writable = lambda : True
buf.write = out.write
ioWrapper = TextIOWrapper(buf, encoding=encoding, errors='xmlcharrefreplace', newline='\n')
self._write = ioWrapper.write
self._flush = ioWrapper.flush
self._ns_contexts = [{}]
self._current_context = self._ns_contexts[-1]
self._undeclared_ns_maps = []
self._encoding = encoding
def unpack(self, buffer: io.BufferedIOBase):
data = self.STRUCT.unpack(buffer)
layout_offset = buffer.tell()
buffer.seek(data.pop("__member_offset"))
struct_offset = 0
self.members = []
for _ in range(data.pop("__member_count")):
member = LayoutMember(buffer, dict(struct_offset=struct_offset))
self.members.append(member)
struct_offset += member.layout_type.size()
buffer.seek(layout_offset)
def __init__(self,
name,
mode="r",
buffer_size=None,
max_buffers=0,
max_workers=None,
**kwargs):
if "a" in mode:
raise NotImplementedError('"a" mode not implemented yet')
BufferedIOBase.__init__(self)
ObjectIOBase.__init__(self, name, mode=mode)
WorkerPoolBase.__init__(self, max_workers)
self._raw = self._RAW_CLASS(name, mode=mode, **kwargs)
self._raw._is_raw_of_buffered = True
self._mode = self._raw.mode
self._name = self._raw.name
self._client_kwargs = self._raw._client_kwargs
if not buffer_size or buffer_size < 0:
self._buffer_size = self.DEFAULT_BUFFER_SIZE
elif buffer_size < self.MINIMUM_BUFFER_SIZE:
self._buffer_size = self.MINIMUM_BUFFER_SIZE
elif self.MAXIMUM_BUFFER_SIZE and buffer_size > self.MAXIMUM_BUFFER_SIZE:
self._buffer_size = self.MAXIMUM_BUFFER_SIZE
else:
self._buffer_size = buffer_size
if self._writable:
self._max_buffers = max_buffers
self._buffer_seek = 0
self._write_buffer = bytearray(self._buffer_size)
self._seekable = False
self._write_futures = []
self._raw_flush = self._raw._flush
# Size used only with random write access
# Value will be lazy evaluated latter if needed.
self._size_synched = False
self._size = 0
self._size_lock = Lock()
else:
self._size = self._raw._size
self._read_range = self.raw._read_range
self._seekable = self.raw._seekable
if max_buffers:
self._max_buffers = max_buffers
else:
self._max_buffers = ceil(self._size / self._buffer_size)
self._read_queue = dict()
def _extract_next_file(
archive_file: io.BufferedIOBase) -> Iterator[Tuple[str, bytes]]:
"""Extracts the next available file from the archive.
Reads the next available file header section and yields its filename and
content in bytes as a tuple. Stops when there are no more available files in
the provided archive_file.
Args:
archive_file: The archive file object, of which cursor is pointing to the
next available file header section.
Yields:
The name and content of the next available file in the given archive file.
Raises:
RuntimeError: The archive_file is in an unknown format.
"""
while True:
header = archive_file.read(60)
if not header:
return
elif len(header) < 60:
raise RuntimeError('Invalid file header format.')
# For the details of the file header format, see:
# https://en.wikipedia.org/wiki/Ar_(Unix)#File_header
# We only need the file name and the size values.
name, _, _, _, _, size, end = struct.unpack('=16s12s6s6s8s10s2s',
header)
if end != b'`\n':
raise RuntimeError('Invalid file header format.')
# Convert the bytes into more natural types.
name = name.decode('ascii').strip()
size = int(size, base=10)
odd_size = size % 2 == 1
# Handle the extended filename scheme.
if name.startswith('#1/'):
filename_size = int(name[3:])
name = archive_file.read(filename_size).decode('utf-8').strip(
' \x00')
size -= filename_size
file_content = archive_file.read(size)
# The file contents are always 2 byte aligned, and 1 byte is padded at the
# end in case the size is odd.
if odd_size:
archive_file.read(1)
yield (name, file_content)
def __init__(self,
stream_name: str,
partition_key: str,
*,
chunk_size: int = DEFAULT_CHUNK_SIZE,
encoder: str = 'utf-8',
workers: int = 1,
**boto_session_kwargs):
"""
:param stream_name: Name of the Kinesis stream
:type stream_name: str
:param partition_key: Kinesis partition key used to group data by shards
:type partition_key: str
:param chunk_size: the size of a a chunk of records for rotation threshold (default 524288)
:type chunk_size: int
:param encoder: the encoder to be used for log records (default 'utf-8')
:type encoder: str
:param workers: the number of background workers that rotate log records (default 1)
:type workers: int
:param boto_session_kwargs: additional keyword arguments for the AWS Kinesis Resource
:type boto_session_kwargs: boto3 resource keyword arguments
"""
self._client = client('kinesis', **boto_session_kwargs)
self.chunk_size = chunk_size
self.stream_name = stream_name
self.tasks = Queue()
self.partition_key = partition_key
self.encoder = encoder
try:
stream_desc = self._client.describe_stream(
StreamName=self.stream_name)
if stream_desc['StreamDescription']['StreamStatus'] != 'ACTIVE':
raise AssertionError
except Exception:
raise ValueError(
'Kinesis stream %s does not exist or inactive, or insufficient permissions'
% stream_name)
self.workers = [
threading.Thread(target=task_worker,
args=(self.tasks, ),
daemon=True).start()
for _ in range(int(max(workers, MIN_WORKERS_NUM) / 2) + 1)
]
self._stream = BytesIO()
self._is_open = True
BufferedIOBase.__init__(self)
def write(self, stream: BufferedIOBase, nodes: List[SceneNode], mode = MeshWriter.OutputMode.BinaryMode) -> bool:
if mode != MeshWriter.OutputMode.BinaryMode:
Logger.log("e", "GCodeGzWriter does not support text mode.")
return False
#Get the g-code from the g-code writer.
gcode_textio = StringIO() #We have to convert the g-code into bytes.
success = PluginRegistry.getInstance().getPluginObject("GCodeWriter").write(gcode_textio, None)
if not success: #Writing the g-code failed. Then I can also not write the gzipped g-code.
return False
result = gzip.compress(gcode_textio.getvalue().encode("utf-8"))
stream.write(result)
return True
def send_message(
wfile: BufferedIOBase,
payload: bytes,
opcode=OPCODE["text"],
rsv1=0,
rsv2=0,
rsv3=0,
):
if len(payload) > 0x7FFFFFFFFFFFFFFF:
raise ValueError(
"Payload to big. Sending fragmented messages not implemented."
)
frame = _encode_data_frame(1, opcode, rsv1, rsv2, rsv3, payload)
wfile.write(frame)
def merge_sort_stupid(fin: io.BufferedIOBase, fout: io.BufferedIOBase, memory_size: int, left=0, count=None):
fout.seek(0)
if count is None:
count = content_length(fin, preserve_pos=False)
if count <= memory_size:
go_to_pos(fin, left)
write_content(fout, sorted(read_content(fin, count=count)), batch_size=memory_size)
return
with tmp_file() as left_f, tmp_file() as right_f:
merge_sort_stupid(fin, left_f, memory_size, left, count=count // 2)
merge_sort_stupid(fin, right_f, memory_size, left + count // 2, count=count - count // 2)
left_f.seek(0)
right_f.seek(0)
write_content(fout, heapq.merge(read_content(left_f, batch_size=memory_size // 2),
read_content(right_f, batch_size=memory_size // 2)),
batch_size=memory_size)
def _check_sorted(self, source: io.BufferedIOBase, res: io.BufferedIOBase):
hashes_size = 2**20
def h(value):
return hash(value) % hashes_size
source.seek(0)
source_content = list(itertools.repeat(0, hashes_size))
for v in read_content(source):
source_content[h(v)] += 1
res.seek(0)
res_content = list(itertools.repeat(0, hashes_size))
prev = None
for cur in read_content(res):
res_content[h(cur)] += 1
self.assertTrue(prev is None or prev <= cur)
prev = cur
self.assertTrue(source_content == res_content, 'Content differs')
def read_content(f: io.BufferedIOBase, count=None, batch_size=_DEFAULT_BATCH_SIZE):
while True:
values_to_read = batch_size if count is None else min(count, batch_size)
b = f.read(values_to_read * _VALUE_SIZE)
if not b:
return
arr = _make_array()
arr.frombytes(b)
yield from arr
if count is not None:
count -= len(arr)
if count == 0:
return
def content_length(f: io.BufferedIOBase, preserve_pos=True):
if preserve_pos:
pos = f.tell()
f.seek(0, io.SEEK_END)
res = f.tell() // _VALUE_SIZE
if preserve_pos:
f.seek(pos)
return res
def go_to_pos(f: io.BufferedIOBase, i: int):
f.seek(i * _VALUE_SIZE)
def close(self):
if not self.coroutine_result:
result = call_result(self.coroutine.throw, EOFError, EOFError())
if isinstance(result, RaiseResult):
self.coroutine_result = result.generator_result()
return BufferedIOBase.close(self)
def byte_object(self, file: BufferedIOBase):
file.write(self.key)
file.write(len(self.data).to_bytes(len(self.key), 'little'))
file.write(self.data)
def __init__(self, stream: io.BufferedIOBase, new: Version = None):
"""Parse a MIX from `stream`, which must be a buffered file object.
If `new` is not None, initialize an empty MIX of this version instead.
MixParseError is raised on parsing errors.
"""
# Initialize mandatory attributes
self._dirty = False
self._stream = None
self._open = []
# If stream is, for example, a raw I/O object, files could be destroyed
# without ever raising an error, so check this.
if not isinstance(stream, io.BufferedIOBase):
raise TypeError("`stream` must be an instance of io.BufferedIOBase")
if not stream.readable():
raise ValueError("`stream` must be readable")
if not stream.seekable():
raise ValueError("`stream` must be seekable")
if new is not None:
# Start empty (new file)
if type(new) is not Version:
raise TypeError("`new` must be a Version enumeration member or None")
if version is Version.RG:
raise NotImplementedError("RG MIX files are not yet supported")
self._stream = stream
self._index = {}
self._contents = []
self._version = version
self._flags = 0
return
# Parse an existing file
filesize = stream.seek(0, io.SEEK_END)
if filesize <= 6:
raise MixParseError("File too small")
stream.seek(0)
first4 = stream.read(4)
if first4 == b"MIX1":
raise NotImplementedError("RG MIX files are not yet supported")
elif first4[:2] == b"\x00\x00":
# It seems we have a RA or TS MIX so check the flags
flags = int.from_bytes(first4[2:], "little")
if flags > 3:
raise MixParseError("Unsupported properties")
if flags & 2:
raise NotImplementedError("Encrypted MIX files are not yet supported")
# FIXME HERE: 80 bytes of westwood key_source if encrypted,
# to create a 56 byte long blowfish key from it.
#
# They are followed by a number of 8 byte blocks,
# the first of them decrypting to filecount and bodysize.
# Encrypted TS MIXes have a key.ini we can check for later,
# so at this point assume Version.TS only if unencrypted.
# Stock RA MIXes seem to be always encrypted.
version = Version.TS
# RA/TS MIXes hold their filecount after the flags,
# whilst for TD MIXes their first two bytes are the filecount.
filecount = int.from_bytes(stream.read(2), "little")
else:
version = Version.TD
flags = 0
filecount = int.from_bytes(first4[:2], "little")
stream.seek(2)
# From here it's the same for every unencrypted MIX
bodysize = int.from_bytes(stream.read(4), "little")
indexoffset = stream.tell()
indexsize = filecount * 12
bodyoffset = indexoffset + indexsize
# Check if data is sane
# FIXME: Checksummed MIXes have 20 additional bytes after the body.
if filesize - bodyoffset != bodysize:
raise MixParseError("Incorrect filesize or invalid header")
# OK, time to read the index
index = {}
for key, offset, size in struct.iter_unpack("<LLL", stream.read(indexsize)):
offset += bodyoffset
if offset + size > filesize:
raise MixParseError("Content extends beyond end of file")
index[key] = _MixNode(key, offset, size, size, None)
if len(index) != filecount:
raise MixParseError("Duplicate key")
# Now read the names
# TD/RA: 1422054725; TS: 913179935
for dbkey in (1422054725, 913179935):
if dbkey in index:
stream.seek(index[dbkey].offset)
header = stream.read(32)
if header != b"XCC by Olaf van der Spek\x1a\x04\x17'\x10\x19\x80\x00":
continue
dbsize = int.from_bytes(stream.read(4), "little") # Total filesize
if dbsize != index[dbkey].size or dbsize > 16777216:
raise MixParseError("Invalid name table")
# Skip four bytes for XCC type; 0 for LMD, 2 for XIF
# Skip four bytes for DB version; Always zero
stream.seek(8, io.SEEK_CUR)
gameid = int.from_bytes(stream.read(4), "little") # XCC Game ID
# XCC saves alias numbers, so converting them
# to `Version` is not straight forward.
# FIXME: Check if Dune games and Nox also use MIX files
if gameid == 0:
if version is not Version.TD:
continue
elif gameid == 1:
version = Version.RA
elif 2 <= gameid <= 6 or gameid == 15:
version = Version.TS
else:
continue
namecount = int.from_bytes(stream.read(4), "little")
bodysize = dbsize - 53 # Size - Header - Last byte
namelist = stream.read(bodysize).split(b"\x00") if bodysize else []
if len(namelist) != namecount:
raise MixParseError("Invalid name table")
# Remove Database from index
del index[dbkey]
# Add names to index
names = False
for name in namelist:
name = name.decode(ENCODING, "ignore")
key = genkey(name, version)
if key in index:
index[key].name = name
names = True
# XCC sometimes puts two Databases in a file by mistake,
# so if no names were found, give it another try
if names: break
# Create a sorted list of all contents
contents = sorted(index.values(), key=lambda node: node.offset)
# Calculate alloc values
# This is the size up to wich a file may grow without needing a move
for i in range(len(contents) - 1):
contents[i].alloc = contents[i+1].offset - contents[i].offset
if contents[i].alloc < contents[i].size:
raise MixParseError("Overlapping file boundaries")
# Populate the object
self._stream = stream
self._version = version
self._index = index
self._contents = contents
self._flags = flags
def __init__(self):
self.coroutine_result = None
self.buffer = bytes()
return BufferedIOBase.__init__(self)
