class SandboxInOut(object):
"""Collect written text, and return it when called."""
class InnerSandboxInOut(object):
def __init__(self, printer, _getattr_=None):
self.printer = printer
self._getattr_ = _getattr_
def write(self, text: str):
self.printer.write(text, consumer=False)
def read(self):
line = self.printer.readline(consumer=True)
if len(line) == 0:
return None
return str(line)
def __call__(self, testing=None):
return self
def _call_print(self, *objects, **kwargs):
if kwargs.get('file', None) is None:
kwargs['file'] = self
else:
self._getattr_(kwargs['file'], 'write')
print(*objects, **kwargs)
def __init__(self, _getattr_=None):
self.stream = BufferedRandom(BytesIO())
self.position = 0
def printer(self):
return self.InnerSandboxInOut(self)
def reader(self):
return lambda: self.readline()
def write(self, line, consumer=True):
if not consumer:
self.stream.seek(2) # go to the end of the stream
else:
self.stream.seek(0, self.position)
self.stream.write(bytes(line, 'utf-8'))
if consumer:
self.position = self.stream.tell()
def readline(self, consumer=True):
if not consumer:
self.stream.seek(2) # go to the end of the stream
else:
self.stream.seek(0, self.position)
line = self.stream.readline()
if len(line) == 0:
return None
if consumer:
self.position = self.stream.tell()
return str(line.decode('utf-8')).strip()
def get_stream(self):
return self.stream
def saveWorld(self, world: World, fileIO: io.BufferedRandom,
worldManager: WorldManager):
# Checking if file size matches!
if not (world.sizeX == 256 and world.sizeY == 256
and world.sizeZ == 256):
raise WorldFormatError(
f"RawWorldFormat - Trying to save world that has invalid world size! Expected: 256, 256, 256! Got: {world.sizeX}, {world.sizeY}, {world.sizeZ}!"
)
# Clearing Current Save File
fileIO.truncate(0)
fileIO.seek(0)
# Saving Map To File
fileIO.write(world.gzipMap())
def __init__(self, file_: BufferedRandom, length: int) -> None:
# Read the file into a buffer.
file_.seek(0, SEEK_END)
buf = bytearray(file_.tell())
file_.seek(0, SEEK_SET)
file_.readinto(buf)
# Overwrite the header, in case it is currently being written to.
header = list_header(length)
buf[:len(header)] = header
# Parse the buffer's contents as list items.
super().__init__(namespacify(cbor2.loads(buf)))
# Store the file pointer for `extend` calls.
self._file = file_
def merge_pdf(pages, target_pdf, img_format='png'):
merged_pdf = PyPDF2.PdfFileWriter()
log.debug('Merging {} into one PDF document'.format(len(pages)))
for page in pages:
# reset binary stream position
page.seek(0)
pdf_page_stream = BufferedRandom(BytesIO())
log.debug('Converting image to PDF')
with wandImage(file=page, format=img_format) as image:
img_converted = image.convert('pdf')
img_converted.save(file=pdf_page_stream)
img_converted.close()
log.debug('Adding PDF page to merged document')
pdf_page_stream.seek(0)
merged_pdf.addPage(PyPDF2.PdfFileReader(pdf_page_stream).getPage(0))
merged_pdf.write(open(target_pdf, 'wb'))
def merge_pdf(pages, target_pdf, img_format='png'):
merged_pdf = PyPDF2.PdfFileWriter()
log.debug('Merging {} into one PDF document'.format(len(pages)))
for page in pages:
# reset binary stream position
page.seek(0)
pdf_page_stream = BufferedRandom(BytesIO())
log.debug('Converting image to PDF')
with wandImage(file=page, format=img_format) as image:
img_converted = image.convert('pdf')
img_converted.save(file=pdf_page_stream)
img_converted.close()
log.debug('Adding PDF page to merged document')
pdf_page_stream.seek(0)
merged_pdf.addPage(PyPDF2.PdfFileReader(pdf_page_stream).getPage(0))
merged_pdf.write(open(target_pdf, 'wb'))
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'):
pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb'))
logging.debug('Source PDF {} contains {} pages'.format(
src_filename, pdf_source.getNumPages()))
pages = []
if not page_list:
log.debug('Extracting all pages from PDF')
# no pages defined, so create a list of all page numbers
page_list = list(range(pdf_source.getNumPages()))
else:
log.debug('Extracting {} page(s) from PDF'.format(len(page_list)))
for page_nr in page_list:
log.debug('Processing page {}'.format(page_nr))
page = pdf_source.getPage(page_nr)
log.debug('Extracting page from source PDF')
# extract single page and save it to a temporary stream
pdf_writer = PyPDF2.PdfFileWriter()
pdf_writer.addPage(page)
pdf_page_stream = BufferedRandom(BytesIO())
pdf_writer.write(pdf_page_stream)
# reset the binary stream's position to the beginning
pdf_page_stream.seek(0)
log.debug('Converting PDF page to image ({})'.format(img_format))
# Define the resolution when opening the intermediate PDF for better quality converted PNGs
# http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion
with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page:
image_page = pdf_page.convert(img_format)
image_page_stream = BufferedRandom(BytesIO())
image_page.save(file=image_page_stream)
pages.append(image_page_stream)
return pages
def split_pdf(src_filename, pdf_res, page_list=None, img_format='png'):
pdf_source = PyPDF2.PdfFileReader(open(src_filename, 'rb'))
logging.debug('Source PDF {} contains {} pages'.format(src_filename, pdf_source.getNumPages()))
pages = []
if not page_list:
log.debug('Extracting all pages from PDF')
# no pages defined, so create a list of all page numbers
page_list = list(range(pdf_source.getNumPages()))
else:
log.debug('Extracting {} page(s) from PDF'.format(len(page_list)))
for page_nr in page_list:
log.debug('Processing page {}'.format(page_nr))
page = pdf_source.getPage(page_nr)
log.debug('Extracting page from source PDF')
# extract single page and save it to a temporary stream
pdf_writer = PyPDF2.PdfFileWriter()
pdf_writer.addPage(page)
pdf_page_stream = BufferedRandom(BytesIO())
pdf_writer.write(pdf_page_stream)
# reset the binary stream's position to the beginning
pdf_page_stream.seek(0)
log.debug('Converting PDF page to image ({})'.format(img_format))
# Define the resolution when opening the intermediate PDF for better quality converted PNGs
# http://stackoverflow.com/questions/17314382/improve-quality-of-wand-conversion
with wandImage(file=pdf_page_stream, resolution=pdf_res) as pdf_page:
image_page = pdf_page.convert(img_format)
image_page_stream = BufferedRandom(BytesIO())
image_page.save(file=image_page_stream)
pages.append(image_page_stream)
return pages
def read_frame_data(seekable_binary_stream: io.BufferedRandom):
seekable_binary_stream.seek(0)
header = seekable_binary_stream.read(2)
if header != b'\xff\xd8':
raise Exception
size = None
while size is None:
marker = seekable_binary_stream.read(2)
# fix marker reading position
if marker[0] != 255 and marker[1] == 255:
seekable_binary_stream.seek(-1, 1)
marker = seekable_binary_stream.read(2)
if marker == b'\xff\x00':
raise ValueError("jpeg marker not found")
if marker in START_OF_FRAME_MARKERS:
seekable_binary_stream.seek(3, 1)
size = struct.unpack('>HH', seekable_binary_stream.read(4))
components = seekable_binary_stream.read(1)[0]
component_scales = []
subsampling = SUPPORTED_COLOR_SPACES.NONE
for component in range(components):
component_id = seekable_binary_stream.read(1)[0]
colorspace_struct = seekable_binary_stream.read(1)[0]
h = (colorspace_struct & 0xf0) >> 4
v = colorspace_struct & 0x0f
component_scales.append((h, v))
seekable_binary_stream.seek(1, 1)
if component_scales[0] == (2, 2):
subsampling = SUPPORTED_COLOR_SPACES.YUV420
elif component_scales[0] == (2, 1) or component_scales[0] == (1,
2):
subsampling = SUPPORTED_COLOR_SPACES.YUV422
else:
subsampling = SUPPORTED_COLOR_SPACES.YUV444
return (size, subsampling)
else:
frame_len = struct.unpack('>H', seekable_binary_stream.read(2))[0]
seekable_binary_stream.seek(frame_len - 2, 1)
def read(self, path, offset, size):
buf = None
br = None
conn = None
try:
conn, desc = self.session.data_objects.open(path, O_RDONLY)
raw = iRODSDataObjectFileRaw(conn, desc)
br = BufferedRandom(raw)
new_offset = br.seek(offset)
if new_offset == offset:
buf = br.read(size)
finally:
if br:
br.close()
if conn:
conn.release(True)
return buf
def FileSeek(file_handle: io.BufferedRandom, offset: int, origin: int) -> bool:
file_handle.seek(offset, origin)
return True
class BinaryCopy():
def __init__(self, chunk_size=256, buffer_size=64 * 1024 * 1024):
self.chunk_size = chunk_size
self.buffer_size = buffer_size
self.fs = BufferedRandom(BytesIO(), buffer_size=buffer_size)
self._row_header_struct = Struct("!hiqiq")
self._row_int_struct = Struct("!i")
self._row_bigint_struct = Struct("!iq")
self._row_null_val = pack("!i", -1)
def write_binary_header(self):
self.fs.write(pack('!11sii', b'PGCOPY\n\xff\r\n\0', 0, 0))
def write_binary_string(self, obj, keyname):
try:
val = obj[keyname]
if val is None:
self.fs.write(self._row_null_val)
else:
val = val.encode()
self.fs.write(self._row_int_struct.pack(len(val)))
self.fs.write(val)
except KeyError as e:
self.fs.write(self._row_null_val)
def write_binary_bigint(self, val):
if val is None:
self.fs.write(self._row_null_val)
else:
self.fs.write(self._row_bigint_struct.pack(8, val))
def write_comment_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
8, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_article(obj.get("link_id", None)))
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
self.write_binary_bigint(get_parent(obj.get("parent_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def write_submission_row(self, row):
obj = row[1]
self.fs.write(
_row_header_struct.pack(
6, 8, int(obj["id"], 36), 8,
timestamp_to_pgtimestamp(int(obj["created_utc"]))))
# write article id
# these have to be written separately because of possible null
self.write_binary_bigint(get_sub_id(obj.get("subreddit_id", None)))
# write strings
self.write_binary_string(obj, "author")
self.write_binary_string(obj, "subreddit")
# write jsonb data
data = row[0].encode()
self.fs.write(pack("!ib", len(data) + 1, 1))
self.fs.write(data)
def copy_comments(self, conn, table, lines):
self.write_binary_header()
for l in lines:
self.write_comment_row(l)
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
def copy(self, conn, table, lines, thing_type):
self.write_binary_header()
if thing_type == "comments":
for l in lines:
self.write_comment_row(l)
elif thing_type == "submissions":
for l in lines:
self.write_submission_row(l)
else:
raise Exception("Unknown thing type {}".format(thing_type))
# write end of task
self.fs.write(pack('!h', -1))
self.fs.flush()
self.fs.seek(0)
conn.cursor.copy_expert("copy %s from stdin with binary " % (table),
self.fs)
self.fs.seek(0)
self.fs.truncate()
class BASE_WS_DEVICE:
def __init__(self,
target,
password,
init=False,
ssl=False,
auth=False,
capath=CA_PATH,
passphrase=None):
self.ws = None
self.ip = target
if ':' in password:
self.pswd, self.passphrase = password.split(':')
else:
self.pswd = password
self.passphrase = passphrase
if ':' in target:
self.ip, self.port = target.split(':')
self.port = int(self.port)
else:
self.port = 8266
self.hostname = None
self.hostname_mdns = None
self.bytes_sent = 0
self.buff = b''
self.raw_buff = b''
self.prompt = b'>>> '
self.response = ''
self._kbi = '\x03'
self._banner = '\x02'
self._reset = '\x04'
self._hreset = "import machine; machine.reset()\r"
self._traceback = b'Traceback (most recent call last):'
self._flush = b''
self.linebuffer = BufferedRandom(BytesIO())
self._debug = False
self.output = None
self.platform = None
self.connected = False
self.repl_CONN = self.connected
self._ssl = ssl
self._uriprotocol = 'ws'
if ssl:
self._uriprotocol = 'wss'
if init:
ip_now = None
if self.passphrase:
auth = True
ssl = True
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.ip.endswith('.local'):
self.hostname = self.ip
self.hostname_mdns = self.ip
try:
ip_now = socket.gethostbyname(self.hostname)
except socket.gaierror:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
if not ssl:
self._uriprotocol = 'ws'
self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}',
self.pswd)
else:
if self.port == 8266:
self.port = 8833
self._uriprotocol = 'wss'
self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}',
self.pswd,
auth=auth,
capath=capath,
passphrase=self.passphrase)
if self.ws:
self.connected = True
# resolve name, store ip in self.ip
# store mdns name in self.hostname
if ip_now:
self.ip = ip_now
self.repl_CONN = self.connected
else:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
def open_wconn(self, ssl=False, auth=False, capath=CA_PATH):
try:
ip_now = None
if self.passphrase:
auth = True
ssl = True
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.ip.endswith('.local'):
self.hostname = self.ip
self.hostname_mdns = self.ip
try:
ip_now = socket.gethostbyname(self.hostname)
except socket.gaierror:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
if not ssl:
self._uriprotocol = 'ws'
if self.port == 8833:
self.port = 8266
self.ws = wsclient.connect(f'ws://{self.ip}:{self.port}',
self.pswd)
else:
self._uriprotocol = 'wss'
if self.port == 8266:
self.port = 8833
if self.passphrase:
auth = True
self.ws = wsclient.connect(f'wss://{self.ip}:{self.port}',
self.pswd,
auth=auth,
capath=capath,
passphrase=self.passphrase)
if self.ws:
self.connected = True
self.repl_CONN = self.connected
if ip_now:
self.ip = ip_now
else:
raise DeviceNotFound(
f"WebSocketDevice @ "
f"{self._uriprotocol}:"
f"//{self.ip}:{self.port} is not reachable")
except sslib.SSLError:
raise sslib.SSLError
except Exception as e:
print(e)
def close_wconn(self):
if self.ws:
self.ws.close()
self.connected = False
if self.hostname_mdns:
self.ip = self.hostname_mdns
self.repl_CONN = self.connected
time.sleep(0.1)
def connect(self, **kargs):
self.open_wconn(**kargs)
def disconnect(self):
self.close_wconn()
@property
def address(self):
return self.ip
@property
def debug(self):
return self._debug
@debug.setter
def debug(self, opt):
assert isinstance(opt, bool)
self._debug = opt
self.ws.debug = opt
def write(self, cmd):
n_bytes = len(bytes(cmd, 'utf-8'))
self.ws.send(cmd)
return n_bytes
def read_all(self):
self.ws.sock.settimeout(None)
try:
self.raw_buff = b''
while self.prompt not in self.raw_buff:
try:
fin, opcode, data = self.ws.read_frame()
self.raw_buff += data
except AttributeError:
pass
return self.raw_buff
except socket.timeout:
return self.raw_buff
def flush(self):
self.ws.sock.settimeout(0.01)
self._flush = b''
self.linebuffer.truncate(0)
self.linebuffer.seek(0)
self.ws.reset_buffers()
while self.ws_readable():
while True:
try:
fin, opcode, data = self.ws.read_frame()
self._flush += data
except socket.timeout:
break
except wsprotocol.NoDataException:
break
self.ws.reset_buffers()
def ws_readable(self):
for i in range(3):
try:
readable, writable, exceptional = select.select([self.ws.sock],
[self.ws.sock],
[self.ws.sock])
if readable:
return True
else:
return False
except Exception as e:
if self.debug:
print(e)
time.sleep(0.01)
def wr_cmd(self,
cmd,
silent=False,
rtn=True,
rtn_resp=False,
long_string=False):
self.output = None
self.response = ''
self.buff = b''
self.flush()
self.bytes_sent = self.write(cmd + '\r')
# time.sleep(0.1)
# self.buff = self.read_all()[self.bytes_sent:]
self.buff = self.read_all()
if self.buff == b'':
# time.sleep(0.1)
self.buff = self.read_all()
# print(self.buff)
# filter command
cmd_filt = bytes(cmd + '\r\n', 'utf-8')
self.buff = self.buff.replace(cmd_filt, b'', 1)
if self._traceback in self.buff:
long_string = True
if long_string:
self.response = self.buff.replace(b'\r', b'').replace(
b'\r\n>>> ', b'').replace(b'>>> ',
b'').decode('utf-8', 'ignore')
else:
self.response = self.buff.replace(b'\r\n', b'').replace(
b'\r\n>>> ', b'').replace(b'>>> ',
b'').decode('utf-8', 'ignore')
if not silent:
if self.response != '\n' and self.response != '':
print(self.response)
else:
self.response = ''
if rtn:
self.get_output()
if self.output == '\n' and self.output == '':
self.output = None
if self.output is None:
if self.response != '' and self.response != '\n':
self.output = self.response
if rtn_resp:
return self.output
def cmd(self, cmd, silent=False, rtn=False, long_string=False):
disconnect_on_end = not self.connected
if not self.connected:
self.open_wconn(ssl=self._ssl, auth=True)
self.wr_cmd(cmd, silent=True, long_string=long_string)
if self.connected:
if disconnect_on_end:
self.close_wconn()
self.get_output()
if not silent:
print(self.response)
if rtn:
return self.output
def reset(self, silent=False, reconnect=True, hr=False):
if not silent:
print('Rebooting device...')
if self.connected:
if not hr:
self.bytes_sent = self.write(self._reset)
else:
self.bytes_sent = self.write(self._hreset)
if self._uriprotocol == 'ws':
time.sleep(0.2)
else:
time.sleep(1)
self.close_wconn()
if reconnect:
time.sleep(1)
while True:
try:
self.open_wconn(ssl=self._ssl, auth=True)
self.wr_cmd(self._banner, silent=True)
break
except Exception:
time.sleep(0.5)
self.ws._close()
self.cmd('')
if not silent:
print('Done!')
else:
self.open_wconn(ssl=self._ssl, auth=True)
if not hr:
self.bytes_sent = self.write(self._reset)
else:
self.bytes_sent = self.write(self._hreset)
if self._uriprotocol == 'ws':
time.sleep(0.2)
else:
time.sleep(1)
self.close_wconn()
if not silent:
print('Done!')
def kbi(self, silent=True, pipe=None, long_string=False):
if self.connected:
if pipe is not None:
self.wr_cmd(self._kbi, silent=silent)
bf_output = self.response.split('Traceback')[0]
traceback = 'Traceback' + self.response.split('Traceback')[1]
if bf_output != '' and bf_output != '\n':
pipe(bf_output)
pipe(traceback, std='stderr')
else:
self.wr_cmd(self._kbi, silent=silent, long_string=long_string)
self.cmd('')
else:
self.cmd(self._kbi, silent=silent)
self.cmd('')
def banner(self, pipe=None):
self.wr_cmd(self._banner, silent=True, long_string=True)
if pipe is None:
print(self.response.replace('\n\n', '\n'))
else:
pipe(self.response.replace('\n\n', '\n'))
def get_output(self):
try:
self.output = ast.literal_eval(self.response)
except Exception:
if 'bytearray' in self.response:
try:
self.output = bytearray(
ast.literal_eval(
self.response.strip().split('bytearray')[1]))
except Exception:
pass
else:
if 'array' in self.response:
try:
arr = ast.literal_eval(
self.response.strip().split('array')[1])
self.output = array(arr[0], arr[1])
except Exception:
pass
pass