def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args,
**kwargs):
rounded_sampling_rate = int(round((self.sampling_rate * 4096) / 44100))
data_out.write(
self.struct.pack(kwargs['end_section_offset'],
rounded_sampling_rate, self.flags.value, self.uk1,
self.size))
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("hw5-receiver")
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
num_bytes = 0
sequenceNumber = 0
while True:
data = sock.recv(homework5.MAX_PACKET)
if not data:
break
header = data[:4]
data = data[4:]
tempNumber = struct.unpack("i", header)[0]
if data[4:] is b'':
break
logger.info("Received %d bytes", len(data))
if tempNumber > sequenceNumber:
sequenceNumber = tempNumber
dest.write(data)
num_bytes += len(data)
dest.flush()
sock.send(struct.pack("i", sequenceNumber))
return num_bytes
def write(self, io_out: BufferedIOBase, v: Optional[Dict[str, Any]], otherfields: Dict[str, Any]) -> None:
# If they didn't specify this tlvstream, it's empty.
if v is None:
return
# Make a tuple of (fieldnum, val_to_bin, val) so we can sort into
# ascending order as TLV spec requires.
def write_raw_val(iobuf: BufferedIOBase, val: Any, otherfields: Dict[str, Any]) -> None:
iobuf.write(val)
def get_value(tup):
"""Get value from num, fun, val tuple"""
return tup[0]
ordered: List[Tuple[int,
Callable[[BufferedIOBase, Any, Dict[str, Any]], None],
Any]] = []
for fieldname in v:
f = self.find_field(fieldname)
if f is None:
# fieldname can be an integer for a raw field.
ordered.append((int(fieldname), write_raw_val, v[fieldname]))
else:
ordered.append((f.number, f.write, v[fieldname]))
ordered.sort(key=get_value)
for typenum, writefunc, val in ordered:
buf = BytesIO()
writefunc(cast(BufferedIOBase, buf), val, otherfields)
BigSizeType.write(io_out, typenum)
BigSizeType.write(io_out, len(buf.getvalue()))
io_out.write(buf.getvalue())
def write(self,
stream: BufferedIOBase,
nodes: List[SceneNode],
mode=MeshWriter.OutputMode.BinaryMode) -> bool:
Logger.log("i", "starting ChituCodeWriter.")
if mode != MeshWriter.OutputMode.TextMode:
Logger.log("e", "ChituCodeWriter does not support non-text mode.")
self.setInformation(
catalog.i18nc(
"@error:not supported",
"ChituCodeWriter does not support non-text mode."))
return False
gcode_textio = StringIO() #We have to convert the g-code into bytes.
gcode_writer = cast(
MeshWriter,
PluginRegistry.getInstance().getPluginObject("GCodeWriter"))
success = gcode_writer.write(gcode_textio, None)
if not success:
self.setInformation(gcode_writer.getInformation())
return False
result = self.modify(gcode_textio.getvalue())
stream.write(result)
Logger.log("i", "ChituWriter done")
return True
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = util.logging.get_logger("project-receiver")
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
num_bytes = 0
while True:
data = sock.recv(util.MAX_PACKET)
if not data:
break
logger.info("Received %d bytes", len(data))
dest.write(data)
num_bytes += len(data)
dest.flush()
return num_bytes
def read_socket_to_file(bufferedReader: BufferedReader, file: BufferedIOBase,
size):
"""
Reads data from a socket to a file, logging progress if data is big.
"""
logProgress = False
if size > big_file_threshold:
logProgress = True
try:
pointer = 0
while pointer < size:
data = bufferedReader.read(min(buffer_size, size - pointer))
file.write(data)
pointer += len(data)
if logProgress:
print("Receiving file",
pointer,
"/",
size,
"bytes received",
end='\r')
if logProgress: print()
except Exception:
print("Error while receiving file.")
def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args,
**kwargs):
rounded_sampling_rate = int(round((self.sampling_rate * 4096) / 48000))
data_out.write(
self.struct.pack(self.sampling_rate, rounded_sampling_rate,
self.volume_level, self.flags.value, self.uk1,
self.uk2, self.size))
def write(self, io_out: BufferedIOBase, v: int, otherfields: Dict[str, Any]) -> None:
binval = struct.pack('>Q', v)
while len(binval) != 0 and binval[0] == 0:
binval = binval[1:]
if len(binval) > self.maxbytes:
raise ValueError('{} exceeds maximum {} capacity'
.format(v, self.name))
io_out.write(binval)
def download_file_from_output_directory(base_url: str, relative_path: str,
output: BufferedIOBase):
url = (base_url if base_url.endswith('/') else base_url +
'/') + relative_path.replace('\\', '/').replace('//', '/')
resp = requests.get(url, stream=True)
for chunk in resp.iter_content(1024):
if chunk:
output.write(chunk)
def uncompress(self, f_in: BufferedIOBase, f_out: BufferedIOBase):
while True:
start = uncompress_num(f_in)
if start is None:
break
byte_len = uncompress_num(f_in)
# print(hex(start), hex(byte_len))
f_out.write(get_bytes(self.dict, start, byte_len))
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
global EXPECTED_SEQUENCE
logger = homework5.logging.get_logger("hw5-receiver")
num_bytes = 0
while True:
try:
data = sock.recv(homework5.MAX_PACKET)
# Kill the process as soon as there is nothing else to send
if not data:
break
# Gather the packet and retrieve the sequence number and data
new_packet = decode_packet(data)
header_only = new_packet[0]
data_only = new_packet[1]
# Check if the packet received is not off
if header_only == EXPECTED_SEQUENCE:
# If the packet received also contains data, then send an ack
if data_only is not None:
# Send an Acknowledgement that the data received corresponds
# to expected value
sock.send(make_packet(EXPECTED_SEQUENCE))
logger.info("Received %d bytes", len(data_only))
dest.write(data_only)
num_bytes += len(data_only)
dest.flush()
# Update the expected sequence if the data that we received
# is the one that was expected
update_sequence()
# If the packet sequence is off, resent the
else:
sock.send(make_packet(EXPECTED_SEQUENCE))
# If there was a timeout, continue
except socket.timeout:
continue
return num_bytes
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("hw5-receiver")
num_bytes = 0
sequence = 0
count = 0
while True:
data = sock.recv(homework5.MAX_PACKET)
#print("\nRECEIVER DATA[0]->", data[:2])
if data[0] == sequence:
sock.send(bytes([sequence]))
#print('\n')
print("RECEVIER SENT THESE BYTES->", str(data[0]))
buff = data[1:]
logger.info("Received %d bytes", len(buff))
#print('\n')
dest.write(buff)
count = 0
num_bytes += len(buff)
dest.flush()
if sequence == 255:
sequence = 0
else:
sequence+=1
continue
elif data[:2] == FIN:
sock.send(FIN)
# print("RECV CLOSED SUCCESSFULLY")
sock.close()
break
if data[0] != sequence:
# sequence-=1
# if count ==0:
sock.send(bytes([data[0]]))
print("WRONG SEQUENCE -- RECEVIER GOT THESE BYTES->", str(data[0]))
print("looking for these bytes->",sequence)
# count+=1
# sock.send(bytes([data[0]]))
# print("DATA was dropped I am RECIEVER and still looking for sequence",str(bytes([sequence])))
continue
break
return num_bytes
def compress_num(nums: list, f_out: BufferedIOBase):
# print(hex(nums[0]), hex(nums[1]))
# use most significant bit to tell the end of a number
buf = bytearray()
for num in nums:
while num >= 128:
buf.append(num & 0x7F)
num >>= 7
else:
buf.append(num + 128)
f_out.write(buf)
def transfer_destination_io(self,
*,
source: str,
destination: io.BufferedIOBase,
chunk_size: int = 4096) -> None:
"""Transfer from source file to destination IO.
Note that this can't use std{in,out}=open(...) due to LP #1849753.
:param source: The source path, prefixed with <name:> for a path inside
the instance.
:param destination: An IO stream to write to.
:raises MultipassError: On error.
"""
command = [str(self.multipass_path), "transfer", source, "-"]
proc = subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
)
# Should never happen, but pyright makes noise.
assert proc.stdout is not None
while True:
written = proc.stdout.read(chunk_size)
if written:
destination.write(written)
if len(written) < chunk_size:
logger.debug("Finished streaming standard output")
break
while True:
try:
out, _ = proc.communicate(timeout=1)
except subprocess.TimeoutExpired:
continue
if out:
destination.write(out)
if proc.returncode == 0:
logger.debug("Process completed")
break
if proc.returncode is not None:
raise MultipassError(
command=command,
returncode=proc.returncode,
msg=f"Failed to transfer file {source!r} to destination.",
)
def write(self, stream: BufferedIOBase, nodes: List[SceneNode], mode = MeshWriter.OutputMode.BinaryMode) -> bool:
if mode != MeshWriter.OutputMode.BinaryMode:
Logger.log("e", "GCodeGzWriter does not support text mode.")
return False
#Get the g-code from the g-code writer.
gcode_textio = StringIO() #We have to convert the g-code into bytes.
success = PluginRegistry.getInstance().getPluginObject("GCodeWriter").write(gcode_textio, None)
if not success: #Writing the g-code failed. Then I can also not write the gzipped g-code.
return False
result = gzip.compress(gcode_textio.getvalue().encode("utf-8"))
stream.write(result)
return True
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("reliable-receiver")
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
reply = packet_listening(sock)
acknowledgement = 1
num_bytes = 0
reply1 = []
reply1 = list(reply)
total = 0
for x in range(1):
reply1[0] = random.randint(1,5)
reply1[1] = reply1[1] + 1
reply1[2] = reply1[2] + 1
second_packet = make_packet(reply1[0], reply1[1], reply1[2], reply1[3])
sock.send(second_packet)
a = 2
while(a == 2):
for x in range(1):
reply1[0] = random.randint(1,5)
reply1[1] = reply1[1] + 1
reply1[2] = reply1[2] + 1
second_packet = make_packet(reply1[0], reply1[1], reply1[2], reply1[3])
data1 = sock.recv(5000)
dlen = len(data1) - 8
data = data1[:dlen]
checksum = struct.unpack("L", data1[dlen:])
checkchecksum = binascii.crc32(data)
if data and checkchecksum in checksum:
sock.send(second_packet)
else:
sock.close()
break
logger.info("Received %d bytes", len(data))
dest.write(data)
num_bytes += len(data)
dest.flush()
return num_bytes
def send_message(
wfile: BufferedIOBase,
payload: bytes,
opcode=OPCODE["text"],
rsv1=0,
rsv2=0,
rsv3=0,
):
if len(payload) > 0x7FFFFFFFFFFFFFFF:
raise ValueError(
"Payload to big. Sending fragmented messages not implemented."
)
frame = _encode_data_frame(1, opcode, rsv1, rsv2, rsv3, payload)
wfile.write(frame)
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
num_bytes = 0
received_chunks = []
expected_chunk = 0
last_ack = 0
while True:
# print('Received: ', received_chunks)
data = sock.recv(homework5.MAX_PACKET)
if not data:
break
# print("Received bytes: ", data[0], " -> ", len(data))
# The chunk was not received yet
# print('expected_chunk: ', expected_chunk, 'data[0]; ', data[0])
if data[0] not in received_chunks and data[0] == expected_chunk:
expected_chunk = expected_chunk + 1
expected_chunk = expected_chunk % 256
received_chunks.append(data[0])
if len(received_chunks) > 240:
received_chunks.pop(0)
dest.write(data[1:])
num_bytes += len(data[1:])
dest.flush()
# send ACT
# print('20. sending ack: ', data[0])
sock.send(bytes([data[0]]))
last_ack = data[0]
# ack lost, resent
elif data[0] in received_chunks and rcv_bef(data[0], expected_chunk):
# print('21. sending ack: ', data[0])
sock.send(bytes([data[0]]))
# Chunk was received before
else:
# print('22. sending ack: ', last_ack)
sock.send(bytes([last_ack]))
return num_bytes
def transfer_destination_io(
self, *, source: str, destination: io.BufferedIOBase, chunk_size: int = 4096
) -> None:
"""Transfer from source file to destination IO.
Note that this can't use std{in,out}=open(...) due to LP #1849753.
:param source: The source path, prefixed with <name:> for a path inside
the instance.
:param destination: An IO stream to write to.
:param chunk_size: Number of bytes to transfer at a time. Defaults to
4096.
:raises MultipassError: On error.
"""
command = [str(self.multipass_path), "transfer", source, "-"]
proc = subprocess.Popen(
command,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
# Should never happen, but mypy/pyright makes noise.
assert proc.stdout is not None
assert proc.stderr is not None
while True:
data = proc.stdout.read(chunk_size)
if not data:
break
destination.write(data)
try:
_, stderr = proc.communicate(timeout=30)
except subprocess.TimeoutExpired:
proc.kill()
_, stderr = proc.communicate()
if proc.returncode != 0:
raise MultipassError(
brief=f"Failed to transfer file {source!r}.",
details=errors.details_from_command_error(
cmd=command, stderr=stderr, returncode=proc.returncode
),
)
def write(io_out: BufferedIOBase, v: int, otherfields: Dict[str, Any] = {}) -> None:
if v < 253:
io_out.write(bytes([v]))
elif v < 2**16:
io_out.write(bytes([253]) + struct.pack('>H', v))
elif v < 2**32:
io_out.write(bytes([254]) + struct.pack('>I', v))
else:
io_out.write(bytes([255]) + struct.pack('>Q', v))
def serialize(self, data_out: BufferedIOBase, conf: Configuration, *args,
**kwargs):
level_sfx_groups: LevelSFXContainer = kwargs['level_sfx_groups']
mapping = np.full((self.n_unique_level_sfx, 16), 255, np.uint8)
for group_id, group in enumerate(
level_sfx_groups): # type: int, LevelSFXGroupContainer
channel_id = self.channel_group_mapping[group_id]
count = Counter()
for sound_id, sound in enumerate(group): # type: int, EffectSound
sound_hash = hash(sound.vag.data)
unique_sound_id = [
i for i, v in enumerate(self.sounds_hashes)
if v == sound_hash
][count[sound_hash]]
count[sound_hash] += 1
mapping[unique_sound_id][channel_id] = sound_id
data_out.write(bytes(mapping.flatten()))
def write(self,
stream: BufferedIOBase,
nodes: List[SceneNode],
mode=MeshWriter.OutputMode.BinaryMode) -> bool:
if mode != MeshWriter.OutputMode.BinaryMode:
Logger.log("e", "GCodeGzWriter does not support text mode.")
return False
#Get the g-code from the g-code writer.
gcode_textio = StringIO() #We have to convert the g-code into bytes.
success = PluginRegistry.getInstance().getPluginObject(
"GCodeWriter").write(gcode_textio, None)
if not success: #Writing the g-code failed. Then I can also not write the gzipped g-code.
return False
result = gzip.compress(gcode_textio.getvalue().encode("utf-8"))
stream.write(result)
return True
def copy_byte_range(
infile: io.BufferedIOBase,
outfile: io.BufferedIOBase,
start: int = None,
stop: int = None,
bufsize: int = 16 * 1024,
):
"""Like shutil.copyfileobj, but only copy a range of the streams.
Both start and stop are inclusive.
"""
if start is not None:
infile.seek(start)
while True:
to_read = min(bufsize, stop + 1 - infile.tell() if stop else bufsize)
buf = infile.read(to_read)
if not buf:
break
outfile.write(buf)
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
logger = homework5.logging.get_logger("hw5-receiver")
call = 0
num_bytes = 0
while True:
rcvpkt = sock.recv(homework5.MAX_PACKET + 3)
if not rcvpkt:
break
seqNum, checksum = extract_header(rcvpkt)
corrupt = is_corrupt(rcvpkt)
# Verify received data packet
if not corrupt and seqNum == call:
# Valid packet, send correct ACK
sndpkt = make_ACK(seqNum)
sock.send(sndpkt)
data = extract_data(rcvpkt)
dest.write(data)
num_bytes += len(data)
dest.flush()
call = 1 - call
elif corrupt or seqNum != call:
# Invalid packet, must send wrong ACK
sndpkt = make_ACK(seqNum)
sock.send(sndpkt)
return num_bytes
def pack(self, stream: BufferedIOBase) -> None:
member: ProtoStructMember
for member in self._desc.members:
value = getattr(self, member.name)
if member.arraySize is None:
_pack_type(stream, value, member.type, self._registry)
else:
if member.arraySize != 0:
if len(value) != member.arraySize:
raise SerializationError(
f"Expected {member.arraySize} elements in array, got {len(value)}"
)
else:
if len(value) > 255:
raise SerializationError(
f"Got an array of size {len(value)}. Arrays must not exceed 255 elements"
)
stream.write(pystruct.pack('=B', len(value)))
for element in value:
_pack_type(stream, element, member.type, self._registry)
def recv(sock: socket.socket, dest: io.BufferedIOBase) -> int:
"""
Implementation of the receiving logic for receiving data over a slow,
lossy, constrained network.
Args:
sock -- A socket object, constructed and initialized to communicate
over a simulated lossy network.
Return:
The number of bytes written to the destination.
"""
# Naive solution, where we continually read data off the socket
# until we don't receive any more data, and then return.
number_of_bytes = 0
previous_header = []
# continuous loop for receiving data
while True:
# gets packet
packets = sock.recv(homework5.MAX_PACKET)
# if packet == null, exit out of loop
if not packets:
break
else:
# take out the header from the packet
header = packets[0:4]
# send it back to the sender as confirmation
sock.send(header)
# if header is same as prev header, return to top of loop
if checker(previous_header, header):
continue
length_of_packet = len(packets)
# update total bytes received
number_of_bytes += length_of_packet - 4
# update prev header
previous_header = header
# write it into destination
dest.write(packets[4:length_of_packet])
dest.flush()
return number_of_bytes
def __init__(self, out=None, encoding='iso-8859-1', short_empty_elements=False):
xmltodict.XMLGenerator.__bases__.__init__(self)
buf = BufferedIOBase()
buf.writable = lambda : True
buf.write = out.write
ioWrapper = TextIOWrapper(buf, encoding=encoding, errors='xmlcharrefreplace', newline='\n')
self._write = ioWrapper.write
self._flush = ioWrapper.flush
self._ns_contexts = [{}]
self._current_context = self._ns_contexts[-1]
self._undeclared_ns_maps = []
self._encoding = encoding
def write(self, io_out: BufferedIOBase) -> None:
"""Write a Message into its wire format.
Must not have missing fields.
"""
if self.missing_fields():
raise ValueError('Missing fields: {}'
.format(self.missing_fields()))
io_out.write(struct.pack(">H", self.messagetype.number))
for f in self.messagetype.fields:
# Optional fields get val == None. Usually this means they don't
# write anything, but length fields are an exception: they intuit
# their value from other fields.
if f.name in self.fields:
val = self.fields[f.name]
else:
# If this isn't present, and it's marked optional, don't write.
if f.option is not None:
return
val = None
f.fieldtype.write(io_out, val, self.fields)
def write(self,
stream: BufferedIOBase,
nodes: List[SceneNode],
mode=MeshWriter.OutputMode.BinaryMode) -> bool:
"""Writes the gzipped g-code to a stream.
Note that even though the function accepts a collection of nodes, the
entire scene is always written to the file since it is not possible to
separate the g-code for just specific nodes.
:param stream: The stream to write the gzipped g-code to.
:param nodes: This is ignored.
:param mode: Additional information on what type of stream to use. This
must always be binary mode.
:return: Whether the write was successful.
"""
if mode != MeshWriter.OutputMode.BinaryMode:
Logger.log("e", "GCodeGzWriter does not support text mode.")
self.setInformation(
catalog.i18nc("@error:not supported",
"GCodeGzWriter does not support text mode."))
return False
#Get the g-code from the g-code writer.
gcode_textio = StringIO() #We have to convert the g-code into bytes.
gcode_writer = cast(
MeshWriter,
PluginRegistry.getInstance().getPluginObject("GCodeWriter"))
success = gcode_writer.write(gcode_textio, None)
if not success: #Writing the g-code failed. Then I can also not write the gzipped g-code.
self.setInformation(gcode_writer.getInformation())
return False
result = gzip.compress(gcode_textio.getvalue().encode("utf-8"))
stream.write(result)
return True
def readinto(self,
target_stream: io.BufferedIOBase,
read_buffer_size: int = 1024 * 1024,
decode_unicode: bool = False):
"""copy bytes from the ResponseIO to target stream.
Useage: \n
ha = HttpAccess()\n
respio = ha.get_response_stream(\n
'https://www.google.com',\n
headers='''\n
accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8\n
accept-encoding:gzip, deflate, br\n
accept-language:,zh-CN,zh;q=0.9\n
''')\n
with open('filepath', mode='wb') as fs:\n
respio.readinto(fs)\n
"""
if not isinstance(target_stream, io.IOBase):
raise Exception("target stream is not a io object")
if not target_stream.writable():
raise Exception("target stream is not writable")
for bs in self.read_block(read_buffer_size=read_buffer_size,
decode_unicode=decode_unicode):
target_stream.write(bs)
def __init__(self,
out=None,
encoding='iso-8859-1',
short_empty_elements=False):
"""
:param out: output object
:param encoding: output encoding
:param short_empty_elements: unused parameter needed for compatibility
"""
xmltodict.XMLGenerator.__bases__.__init__(self)
buffer = BufferedIOBase()
buffer.writable = lambda: True
buffer.write = out.write
ioWrapper = TextIOWrapper(buffer,
encoding=encoding,
errors='xmlcharrefreplace',
newline='\n')
self._write = ioWrapper.write
self._flush = ioWrapper.flush
self._ns_contexts = [{}]
self._current_context = self._ns_contexts[-1]
self._undeclared_ns_maps = []
self._encoding = encoding
def byte_object(self, file: BufferedIOBase):
file.write(self.key)
file.write(len(self.data).to_bytes(len(self.key), 'little'))
file.write(self.data)
