def __init__keycodes(self):
# Initialize keyboard data determined by capability.
# Build database of int code <=> KEY_NAME.
self._keycodes = get_keyboard_codes()
# Store attributes as: self.KEY_NAME = code.
for key_code, key_name in self._keycodes.items():
setattr(self, key_name, key_code)
# Build database of sequence <=> KEY_NAME.
self._keymap = get_keyboard_sequences(self)
# build set of prefixes of sequences
self._keymap_prefixes = get_leading_prefixes(self._keymap)
self._keyboard_buf = collections.deque()
if self._keyboard_fd is not None:
locale.setlocale(locale.LC_ALL, '')
self._encoding = locale.getpreferredencoding() or 'ascii'
try:
self._keyboard_decoder = codecs.getincrementaldecoder(
self._encoding)()
except LookupError as err:
warnings.warn('LookupError: {0}, fallback to ASCII for '
'keyboard.'.format(err))
self._encoding = 'ascii'
self._keyboard_decoder = codecs.getincrementaldecoder(
self._encoding)()
def __init__(self, path, header, rec_stdin, start_time_offset=0):
self.path = path
self.header = header
self.rec_stdin = rec_stdin
self.start_time_offset = start_time_offset
self.queue = Queue()
self.stdin_decoder = codecs.getincrementaldecoder('UTF-8')('replace')
self.stdout_decoder = codecs.getincrementaldecoder('UTF-8')('replace')
def test_newline_decoder(self):
encodings = ("utf-8", "latin-1", "utf-16", "utf-16-le", "utf-16-be", "utf-32", "utf-32-le", "utf-32-be")
for enc in encodings:
decoder = codecs.getincrementaldecoder(enc)()
decoder = io.IncrementalNewlineDecoder(decoder, translate=True)
self.check_newline_decoder(decoder, enc)
decoder = codecs.getincrementaldecoder("utf-8")()
decoder = io.IncrementalNewlineDecoder(decoder, translate=True)
self.check_newline_decoder_utf8(decoder)
def __init__(self, path, width=None, height=None, header=None, mode='w', buffering=-1):
self.path = path
self.mode = mode
self.buffering = buffering
self.stdin_decoder = codecs.getincrementaldecoder('UTF-8')('replace')
self.stdout_decoder = codecs.getincrementaldecoder('UTF-8')('replace')
if mode == 'w':
self.header = {'version': 2, 'width': width, 'height': height}
self.header.update(header or {})
assert type(self.header['width']) == int, 'width or header missing'
assert type(self.header['height']) == int, 'height or header missing'
else:
self.header = None
def test_newline_decoder(self):
import codecs
decoder = codecs.getincrementaldecoder("utf-8")()
decoder = io.IncrementalNewlineDecoder(decoder, translate=True)
self.assertEquals(decoder.decode(b'\xe8\xa2\x88'), u"\u8888")
self.assertEquals(decoder.decode(b'\xe8'), u"")
self.assertEquals(decoder.decode(b'\xa2'), u"")
self.assertEquals(decoder.decode(b'\x88'), u"\u8888")
self.assertEquals(decoder.decode(b'\xe8'), u"")
self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True)
decoder.setstate((b'', 0))
self.assertEquals(decoder.decode(b'\n'), u"\n")
self.assertEquals(decoder.decode(b'\r'), u"")
self.assertEquals(decoder.decode(b'', final=True), u"\n")
self.assertEquals(decoder.decode(b'\r', final=True), u"\n")
self.assertEquals(decoder.decode(b'\r'), u"")
self.assertEquals(decoder.decode(b'a'), u"\na")
self.assertEquals(decoder.decode(b'\r\r\n'), u"\n\n")
self.assertEquals(decoder.decode(b'\r'), u"")
self.assertEquals(decoder.decode(b'\r'), u"\n")
self.assertEquals(decoder.decode(b'\na'), u"\na")
self.assertEquals(decoder.decode(b'\xe8\xa2\x88\r\n'), u"\u8888\n")
self.assertEquals(decoder.decode(b'\xe8\xa2\x88'), u"\u8888")
self.assertEquals(decoder.decode(b'\n'), u"\n")
self.assertEquals(decoder.decode(b'\xe8\xa2\x88\r'), u"\u8888")
self.assertEquals(decoder.decode(b'\n'), u"\n")
decoder = codecs.getincrementaldecoder("utf-8")()
decoder = io.IncrementalNewlineDecoder(decoder, translate=True)
self.assertEquals(decoder.newlines, None)
decoder.decode(b"abc\n\r")
self.assertEquals(decoder.newlines, u'\n')
decoder.decode(b"\nabc")
self.assertEquals(decoder.newlines, ('\n', '\r\n'))
decoder.decode(b"abc\r")
self.assertEquals(decoder.newlines, ('\n', '\r\n'))
decoder.decode(b"abc")
self.assertEquals(decoder.newlines, ('\r', '\n', '\r\n'))
decoder.decode(b"abc\r")
decoder.reset()
self.assertEquals(decoder.decode(b"abc"), "abc")
self.assertEquals(decoder.newlines, None)
def checkauto(encoding, input="<?xml version='1.0' encoding='x'?>gürk\u20ac"):
# Check stateless decoder
d = codecs.getdecoder("xml")
assert d(input.encode(encoding))[0] == input.replace("'x'", repr(encoding))
# Check stateless decoder with specified encoding
assert d(input.encode(encoding), encoding=encoding)[0] == input.replace("'x'", repr(encoding))
# Check incremental decoder
id = codecs.getincrementaldecoder("xml")()
assert "".join(id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input.replace("'x'", repr(encoding))
# Check incremental decoder with specified encoding
id = codecs.getincrementaldecoder("xml")(encoding)
assert "".join(id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input.replace("'x'", repr(encoding))
def checkdecl(encoding, input="<?xml version='1.0' encoding={encoding!r}?><gürk>\u20ac</gürk>"):
# Check stateless decoder with encoding autodetection
d = codecs.getdecoder("xml")
input = input.format(encoding=encoding)
assert d(input.encode(encoding))[0] == input
# Check stateless decoder with specified encoding
assert d(input.encode(encoding), encoding=encoding)[0] == input
# Check incremental decoder with encoding autodetection
id = codecs.getincrementaldecoder("xml")()
assert "".join(id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input
# Check incremental decoder with specified encoding
id = codecs.getincrementaldecoder("xml")(encoding)
assert "".join(id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input
def testIncrementalDecoder(self):
# Tests derived from Python standard library test/test_codecs.py
incremental_tests = (
(u"python.org", b"python.org"),
(u"python.org.", b"python.org."),
(u"pyth\xf6n.org", b"xn--pythn-mua.org"),
(u"pyth\xf6n.org.", b"xn--pythn-mua.org."),
)
for decoded, encoded in incremental_tests:
if sys.version_info[0] == 2:
self.assertEqual("".join(codecs.iterdecode(encoded, "idna")),
decoded)
else:
self.assertEqual("".join(codecs.iterdecode((bytes([c]) for c in encoded), "idna")),
decoded)
decoder = codecs.getincrementaldecoder("idna")()
self.assertEqual(decoder.decode(b"xn--xam", ), u"")
self.assertEqual(decoder.decode(b"ple-9ta.o", ), u"\xe4xample.")
self.assertEqual(decoder.decode(b"rg"), u"")
self.assertEqual(decoder.decode(b"", True), u"org")
decoder.reset()
self.assertEqual(decoder.decode(b"xn--xam", ), u"")
self.assertEqual(decoder.decode(b"ple-9ta.o", ), u"\xe4xample.")
self.assertEqual(decoder.decode(b"rg."), u"org.")
self.assertEqual(decoder.decode(b"", True), u"")
def decode(self, input, final=False):
# We're doing basically the same as a ``BufferedIncrementalDecoder``,
# but since the buffer is only relevant until the encoding has been detected
# (in which case the buffer of the underlying codec might kick in),
# we're implementing buffering ourselves to avoid some overhead.
if self.decoder is None:
input = self.buffer + input
if self.encoding is None:
self.encoding = _detectencoding(input, final)
if self.encoding is None:
self.buffer = input # retry the complete input on the next call
return "" # no encoding determined yet, so no output
if self.encoding == "xml":
raise ValueError("xml not allowed as encoding name")
self.buffer = b"" # isn't needed any more, as the decoder might keep its own buffer
self.decoder = codecs.getincrementaldecoder(self.encoding)(self._errors)
if self.headerfixed:
return self.decoder.decode(input, final)
# If we haven't fixed the header yet, the content of ``self.buffer`` is a :class:`str` object
buffer = self.buffer
if isinstance(buffer, bytes):
buffer = buffer.decode("ascii")
output = buffer + self.decoder.decode(input, final)
newoutput = _fixencoding(output, self.encoding, final)
if newoutput is None:
self.buffer = output # retry fixing the declaration (but keep the decoded stuff)
return ""
self.headerfixed = True
return newoutput
def __init__(self, transport, encoding=None, errors="strict"):
StreamTransportAdapter.__init__(self, transport)
if encoding is None:
encoding = sys.getfilesystemencoding()
self.encoding = encoding
self.encoder = codecs.getincrementalencoder(encoding)(errors)
self.decoder = codecs.getincrementaldecoder(encoding)(errors)
def posix_shell(chan,channel,width=90,height=40):
from OpsManage.asgi import channel_layer
stdout = list()
begin_time = time.time()
last_write_time = {'last_activity_time':begin_time}
try:
chan.settimeout(0.0)
while True:
try:
x = u(chan.recv(1024))
if len(x) == 0:
channel_layer.send(channel, {'text': json.dumps(['disconnect',smart_unicode('\r\n*** EOF\r\n')]) })
break
now = time.time()
delay = now - last_write_time['last_activity_time']
last_write_time['last_activity_time'] = now
if x == "exit\r\n" or x == "logout\r\n" or x == 'logout':
chan.close()
else:
stdout.append([delay,codecs.getincrementaldecoder('UTF-8')('replace').decode(x)])
channel_layer.send(channel, {'text': json.dumps(['stdout',smart_unicode(x)]) })
except socket.timeout:
pass
except Exception,e:
channel_layer.send(channel, {'text': json.dumps(['stdout','A bug find,You can report it to me' + smart_unicode(e)]) })
finally:
pass
def __init__(self, server, sock, address):
self.server = server
self.client = sock
self.address = address
self.handshaked = False
self.headerbuffer = bytearray()
self.headertoread = 2048
self.fin = 0
self.data = bytearray()
self.opcode = 0
self.hasmask = 0
self.maskarray = None
self.length = 0
self.lengtharray = None
self.index = 0
self.request = None
self.usingssl = False
self.frag_start = False
self.frag_type = BINARY
self.frag_buffer = None
self.frag_decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
self.closed = False
self.sendq = deque()
self.state = HEADERB1
# restrict the size of header and payload for security reasons
self.maxheader = MAXHEADER
self.maxpayload = MAXPAYLOAD
def decodeResponse(self, rep):
""" decode with correct encoding, relies on header """
header = self.header.splitlines()
encoding = "utf8" # default encoding
for line in header:
line = line.lower().replace(" ", "")
if not line.startswith("content-type:") or \
("text" not in line and "application" not in line):
continue
none, delemiter, charset = line.rpartition("charset=")
if delemiter:
charset = charset.split(";")
if charset:
encoding = charset[0]
try:
#self.log.debug("Decoded %s" % encoding )
if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8):
encoding = 'utf-8-sig'
decoder = getincrementaldecoder(encoding)("replace")
rep = decoder.decode(rep, True)
#TODO: html_unescape as default
except LookupError:
self.log.debug("No Decoder found for %s" % encoding)
except Exception:
self.log.debug("Error when decoding string from %s." % encoding)
return rep
def decode(self, text_utf8, text_latex, inputenc=None):
encoding = 'latex+' + inputenc if inputenc else 'latex'
decoder = codecs.getincrementaldecoder(encoding)()
decoded_parts = (
decoder.decode(text_latex_part, final)
for text_latex_part, final in split_input(text_latex))
self.assertEqual(text_utf8, u''.join(decoded_parts))
def run (self) :
pout = os.pipe()
self.on_init()
cmd = ['dialog', '--output-fd', str(pout[1])]
cmd.extend(['--title', 'TITLE'])
cmd.extend(self.get_widget())
trace("> %s" % ' '.join(cmd))
pid = os.fork()
if pid == 0 :
# child
os.close(pout[0])
proc = subprocess.Popen(cmd, close_fds=False)
r = proc.wait()
os.close(pout[1])
sys.exit(r)
else :
# parent
os.close(pout[1])
dc = codecs.getincrementaldecoder('utf-8')(errors='replace')
out = ''
while True :
data = os.read(pout[0], 4096)
if not data :
out += dc.decode(b'', True)
break
out += dc.decode(data, False)
os.close(pout[0])
rpid, r = os.waitpid(pid, 0)
trace(">> %d: '%s'" % (r, out))
return r, out
def __init__(self):
if utf8validator:
self.validator = utf8validator.Utf8Validator()
else:
self.validator = None
decoderclass = codecs.getincrementaldecoder('utf8')
self.decoder = decoderclass()
def set_keyboard_decoder(self, encoding):
try:
self._keyboard_decoder = codecs.getincrementaldecoder(encoding)()
self._encoding = encoding
except Exception, err:
log = logging.getLogger()
log.exception(err)
def decode(self, input, final=False):
# We're doing basically the same as a ``BufferedIncrementalDecoder``,
# but since the buffer is only relevant until the encoding has been
# detected (in which case the buffer of the underlying codec might
# kick in), we're implementing buffering ourselves to avoid some
# overhead.
if self.decoder is None:
input = self.buffer + input
self.encoding = _detectencoding_str(input, final)
if self.encoding is None:
self.buffer = input # retry the complete input on the next call
return u"" # no encoding determined yet, so no output
if self.encoding == "css":
raise ValueError("css not allowed as encoding name")
self.buffer = "" # drop buffer, as the decoder might keep its own
decoder = codecs.getincrementaldecoder(self.encoding)
self.decoder = decoder(self._errors)
if self.headerfixed:
return self.decoder.decode(input, final)
# If we haven't fixed the header yet,
# the content of ``self.buffer`` is a ``unicode`` object
output = self.buffer + self.decoder.decode(input, final)
encoding = self.encoding
if encoding.replace("_", "-").lower() == "utf-8-sig":
encoding = "utf-8"
newoutput = _fixencoding(output, unicode(encoding), final)
if newoutput is None:
# retry fixing the @charset rule (but keep the decoded stuff)
self.buffer = output
return u""
self.headerfixed = True
return newoutput
def decode_response(self, rep):
"""Decode with correct encoding, relies on header."""
header = self.header.splitlines()
encoding = 'utf8' # default encoding
for line in header:
line = line.lower().replace(' ', '')
if (not line.startswith('content-type:') or
('text' not in line and 'application' not in line)):
continue
_, delemiter, charset = line.rpartition('charset=')
if delemiter:
charset = charset.split(';')
if charset:
encoding = charset[0]
try:
# self.log.debug("Decoded {0}".format(encoding))
if lookup(encoding).name == 'utf-8' and rep.startswith(BOM_UTF8):
encoding = 'utf-8-sig'
decoder = getincrementaldecoder(encoding)('replace')
rep = decoder.decode(rep, True)
# TODO: html_unescape as default
except LookupError:
self.log.debug('No Decoder found for {0}'.format(encoding))
except Exception:
self.log.debug(
'Error when decoding string from {0}'.format(encoding))
return rep
def parse_http_html (resp, parser):
"""`parser` need only have two methods: `feed()` and `close()`."""
debug = False # XXX hack
charset = resp.headers.getparam ('charset')
if charset is None:
charset = 'ISO-8859-1'
dec = codecs.getincrementaldecoder (charset) ()
if debug:
f = open ('debug.html', 'w')
while True:
d = resp.read (4096)
if not len (d):
text = dec.decode ('', final=True)
parser.feed (text)
break
if debug:
f.write (d)
text = dec.decode (d)
parser.feed (text)
if debug:
f.close ()
resp.close ()
parser.close ()
return parser
def _get_decoder(self):
make_decoder = codecs.getincrementaldecoder(self._encoding)
decoder = make_decoder(self._errors)
if self._readuniversal:
decoder = IncrementalNewlineDecoder(decoder, self._readtranslate)
self._decoder = decoder
return decoder
def __init__(self,mSocket):
self.handshaked = False
self.headerbuffer = ''
self.headertoread = 2048
self.sock = mSocket
self.fin = 0
self.data = bytearray()
self.opcode = 0
self.hasmask = 0
self.maskarray = None
self.length = 0
self.lengtharray = None
self.index = 0
self.request = None
self.usingssl = False
self.data_ready = False
#self.last_data=""
self.msgB=[]
self.frag_start = False
self.frag_type = BINARY
self.frag_buffer = None
self.frag_decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
self.closed = False
self.sendq = deque()
self.state = HEADERB1
# restrict the size of header and payload for security reasons
self.maxheader = MAXHEADER
self.maxpayload = MAXPAYLOAD
def read_until_semaphore(fd, semaphore=RECV_SEMAPHORE,
encoding='utf8', timeout=10):
"""Read file descriptor ``fd`` until ``semaphore`` is found.
Used to ensure the child process is awake and ready. For timing
tests; without a semaphore, the time to fork() would be (incorrectly)
included in the duration of the test, which can be very length on
continuous integration servers (such as Travis-CI).
"""
# note that when a child process writes xyz\\n, the parent
# process will read xyz\\r\\n -- this is how pseudo terminals
# behave; a virtual terminal requires both carriage return and
# line feed, it is only for convenience that \\n does both.
outp = six.text_type()
decoder = codecs.getincrementaldecoder(encoding)()
semaphore = semaphore.decode('ascii')
while not outp.startswith(semaphore):
try:
_exc = os.read(fd, 1)
except OSError: # Linux EOF
break
if not _exc: # BSD EOF
break
outp += decoder.decode(_exc, final=False)
assert outp.startswith(semaphore), (
'Semaphore not recv before EOF '
'(expected: %r, got: %r)' % (semaphore, outp,))
return outp[len(semaphore):]
def read_message(self):
# Reassemble fragmented messages.
frame = yield from self.read_data_frame()
if frame is None:
return
if frame.opcode == OP_TEXT:
text = True
elif frame.opcode == OP_BINARY:
text = False
else: # frame.opcode == OP_CONT
raise WebSocketProtocolError("Unexpected opcode")
# Shortcut for the common case - no fragmentation
if frame.fin:
return frame.data.decode('utf-8') if text else frame.data
# 5.4. Fragmentation
chunks = []
if text:
decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
append = lambda f: chunks.append(decoder.decode(f.data, f.fin))
else:
append = lambda f: chunks.append(f.data)
append(frame)
while not frame.fin:
frame = yield from self.read_data_frame()
if frame is None:
raise WebSocketProtocolError("Incomplete fragmented message")
if frame.opcode != OP_CONT:
raise WebSocketProtocolError("Unexpected opcode")
append(frame)
return ('' if text else b'').join(chunks)
def test_incremental_decode(self):
self.assertEquals(
"".join(codecs.iterdecode("python.org", "idna")),
u"python.org"
)
self.assertEquals(
"".join(codecs.iterdecode("python.org.", "idna")),
u"python.org."
)
self.assertEquals(
"".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
u"pyth\xf6n.org."
)
self.assertEquals(
"".join(codecs.iterdecode("xn--pythn-mua.org.", "idna")),
u"pyth\xf6n.org."
)
decoder = codecs.getincrementaldecoder("idna")()
self.assertEquals(decoder.decode("xn--xam", ), u"")
self.assertEquals(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
self.assertEquals(decoder.decode(u"rg"), u"")
self.assertEquals(decoder.decode(u"", True), u"org")
decoder.reset()
self.assertEquals(decoder.decode("xn--xam", ), u"")
self.assertEquals(decoder.decode("ple-9ta.o", ), u"\xe4xample.")
self.assertEquals(decoder.decode("rg."), u"org.")
self.assertEquals(decoder.decode("", True), u"")
def __init__(self, encoding, external_id=None, cmd_postfix="\n", suppress_echo=False):
self.id = uuid4().hex
self.decoder = getincrementaldecoder(encoding)()
self.encoder = getencoder(encoding)
self.external_id = external_id
self.cmd_postfix = cmd_postfix
self.suppress_echo = suppress_echo
def spawn_build(self, iface_name):
assert self.child is None
self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
# Group all the child processes so we can kill them easily
def become_group_leader():
os.setpgid(0, 0)
devnull = os.open(os.devnull, os.O_RDONLY)
try:
self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
stdin = devnull,
stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
preexec_fn = become_group_leader)
finally:
os.close(devnull)
import codecs
decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
while True:
yield tasks.InputBlocker(self.child.stdout, 'output from child')
got = os.read(self.child.stdout.fileno(), 100)
chars = decoder.decode(got, final = not got)
self.details.insert_at_end_and_scroll(chars)
if not got: break
self.child.wait()
code = self.child.returncode
self.child = None
if code:
self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
raise SafeException('Build process exited with error status %d' % code)
self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
def process_frame(self, frame):
assert not frame.opcode.iscontrol()
if self.opcode is None:
if frame.opcode is Opcode.CONTINUATION:
raise ParseFailed("unexpected CONTINUATION")
self.opcode = frame.opcode
elif frame.opcode is not Opcode.CONTINUATION:
raise ParseFailed("expected CONTINUATION, got %r" % frame.opcode)
if frame.opcode is Opcode.TEXT:
self.validator = Utf8Validator()
self.decoder = getincrementaldecoder("utf-8")()
finished = frame.frame_finished and frame.message_finished
if self.decoder is not None:
data = self.decode_payload(frame.payload, finished)
else:
data = frame.payload
frame = Frame(self.opcode, data, frame.frame_finished, finished)
if finished:
self.opcode = None
self.decoder = None
return frame
def __init__(self, cmd='/bin/uname', args=(), env=None, cp437=False):
"""
Class initializer.
:param str cmd: full path of command to execute.
:param tuple args: command arguments as tuple.
:param bool cp437: When true, forces decoding of external program as
codepage 437. This is the most common encoding used
by DOS doors.
:param dict env: Environment variables to extend to the sub-process.
You should more than likely specify values for TERM,
PATH, HOME, and LANG.
"""
self._session, self._term = getsession(), getterminal()
self.cmd = cmd
if isinstance(args, tuple):
self.args = (self.cmd,) + args
elif isinstance(args, list):
self.args = [self.cmd, ] + args
else:
raise ValueError('args must be tuple or list')
self.log = logging.getLogger(__name__)
self.env = (env or {}).copy()
self.env.update(
{'LANG': env.get('LANG', 'en_US.UTF-8'),
'TERM': env.get('TERM', self._term.kind),
'PATH': env.get('PATH', get_ini('door', 'path')),
'HOME': env.get('HOME', os.getenv('HOME')),
'LINES': str(self._term.height),
'COLUMNS': str(self._term.width),
})
self.cp437 = cp437
self._utf8_decoder = codecs.getincrementaldecoder('utf8')()
def test_dbcs(self):
# cp949 decoder is simple with only 1 or 2 bytes sequences.
decoder = codecs.getincrementaldecoder('cp949')()
self.assertEqual(decoder.decode('\xc6\xc4\xc0\xcc\xbd'),
u'\ud30c\uc774')
self.assertEqual(decoder.decode('\xe3 \xb8\xb6\xc0\xbb'),
u'\uc36c \ub9c8\uc744')
self.assertEqual(decoder.decode(''), u'')
def __init__(self, *args, **kwargs):
super(ByteStream, self).__init__(*args, **kwargs)
self.utf8_decoder = codecs.getincrementaldecoder("utf-8")("replace")
def __init__(self, *args, **kwargs):
self.parser = Parser()
self.decoder = codecs.getincrementaldecoder('utf_8')()
def test_decode_unicode(self):
# Trying to decode an unicode string should raise a TypeError
for enc in ALL_CJKENCODINGS:
decoder = codecs.getincrementaldecoder(enc)()
self.assertRaises(TypeError, decoder.decode, "")
def test_newline_decoder():
def check_newline_decoding_utf8(decoder):
# UTF-8 specific tests for a newline decoder
def _check_decode(b, s, **kwargs):
# We exercise getstate() / setstate() as well as decode()
state = decoder.getstate()
assert decoder.decode(b, **kwargs) == s
decoder.setstate(state)
assert decoder.decode(b, **kwargs) == s
_check_decode(b'\xe8\xa2\x88', u"\u8888")
_check_decode(b'\xe8', "")
_check_decode(b'\xa2', "")
_check_decode(b'\x88', u"\u8888")
_check_decode(b'\xe8', "")
_check_decode(b'\xa2', "")
_check_decode(b'\x88', u"\u8888")
_check_decode(b'\xe8', "")
raises(UnicodeDecodeError, decoder.decode, b'', final=True)
decoder.reset()
_check_decode(b'\n', "\n")
_check_decode(b'\r', "")
_check_decode(b'', "\n", final=True)
_check_decode(b'\r', "\n", final=True)
_check_decode(b'\r', "")
_check_decode(b'a', "\na")
_check_decode(b'\r\r\n', "\n\n")
_check_decode(b'\r', "")
_check_decode(b'\r', "\n")
_check_decode(b'\na', "\na")
_check_decode(b'\xe8\xa2\x88\r\n', u"\u8888\n")
_check_decode(b'\xe8\xa2\x88', u"\u8888")
_check_decode(b'\n', "\n")
_check_decode(b'\xe8\xa2\x88\r', u"\u8888")
_check_decode(b'\n', "\n")
def check_newline_decoding(decoder, encoding):
result = []
if encoding is not None:
encoder = codecs.getincrementalencoder(encoding)()
def _decode_bytewise(s):
# Decode one byte at a time
for b in encoder.encode(s):
result.append(decoder.decode(b))
else:
encoder = None
def _decode_bytewise(s):
# Decode one char at a time
for c in s:
result.append(decoder.decode(c))
assert decoder.newlines == None
_decode_bytewise(u"abc\n\r")
assert decoder.newlines == '\n'
_decode_bytewise(u"\nabc")
assert decoder.newlines == ('\n', '\r\n')
_decode_bytewise(u"abc\r")
assert decoder.newlines == ('\n', '\r\n')
_decode_bytewise(u"abc")
assert decoder.newlines == ('\r', '\n', '\r\n')
_decode_bytewise(u"abc\r")
assert "".join(result) == "abc\n\nabcabc\nabcabc"
decoder.reset()
input = u"abc"
if encoder is not None:
encoder.reset()
input = encoder.encode(input)
assert decoder.decode(input) == "abc"
assert decoder.newlines is None
encodings = (
# None meaning the IncrementalNewlineDecoder takes unicode input
# rather than bytes input
None,
'utf-8',
'latin-1',
'utf-16',
'utf-16-le',
'utf-16-be',
'utf-32',
'utf-32-le',
'utf-32-be',
)
import codecs
for enc in encodings:
decoder = enc and codecs.getincrementaldecoder(enc)()
decoder = _io.IncrementalNewlineDecoder(decoder, translate=True)
check_newline_decoding(decoder, enc)
decoder = codecs.getincrementaldecoder("utf-8")()
decoder = _io.IncrementalNewlineDecoder(decoder, translate=True)
check_newline_decoding_utf8(decoder)
def __init__(self):
self._buffer = []
self._decoder1 = codecs.getincrementaldecoder('utf-8')()
self._decoder2 = json.JSONDecoder()
self._lasterror = None
def __init__(self, print_line=False, callback=None):
"""Create aggregator object."""
self._buffer = ""
self._print = print_line
self._callback = callback
self._decoder = codecs.getincrementaldecoder("utf-8")("replace")
def handle(cs,
cf=lambda: None,
rf=lambda dt: None,
df=lambda: None,
h_dt=None):
cs.setblocking(0)
threading.current_thread()._cs_q = []
threading.current_thread()._e = False
threading.current_thread()._tl = threading.Lock()
r_hs = False
r_f = 0
r_t = 0
r_dt = bytearray()
r_m = 0
r_ml = None
r_l = 0
r_ll = None
r_i = 0
r_s = 0
r_fs = False
r_ft = BINARY
r_fb = None
r_fd = codecs.getincrementaldecoder("utf-8")(errors="strict")
while (not threading.current_thread()._e):
try:
if (r_hs is False):
dt = (h_dt if h_dt else cs.recv(65535))
if (not dt):
return
if (b"\r\n\r\n" not in dt):
raise RuntimeError("Header too big")
try:
for e in dt.split(b"\r\n\r\n")[0].split(b"\r\n")[1:]:
if (len(e) > 0
and str(e.split(b":")[0], "utf-8").lower()
== "sec-websocket-key"):
threading.current_thread()._cs_q.append((
BINARY,
f"HTTP/1.1 101 Switching Protocols\r\nUpgrade: WebSocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {base64.b64encode(hashlib.sha1(e[len(e.split(b':')[0])+2:]+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11'.encode('ascii')).digest()).decode('ascii')}\r\n\r\n"
.encode("ascii")))
r_hs = True
cf()
break
if (r_hs == False):
raise KeyError
except Exception as e:
traceback.print_exception(None, e, e.__traceback__)
dt = "HTTP/1.1 426 Upgrade Required\r\nUpgrade: WebSocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nContent-Type: text/plain\r\n\r\nThis service requires use of the WebSocket protocol\r\n".encode(
"ascii")
i = 0
l = len(dt)
while (i < l):
try:
j = cs.send(dt[i:])
if (j == 0):
return
i += j
except socket.error as e:
if (e.errno in [errno.EAGAIN, errno.EWOULDBLOCK]):
continue
else:
raise e
cs.close()
raise RuntimeError(f"Handshake failed: {e}")
else:
try:
dt = cs.recv(16384)
if (not dt):
return
for b in dt:
ok = False
if (r_s == 0):
r_f, rsv, r_t = b & 0x80, b & 0x70, b & 0x0f
r_s = 1
r_i = 0
r_l = 0
r_ll = bytearray()
r_dt = bytearray()
if (rsv != 0):
raise RuntimeError("RSV bit must be 0")
elif (r_s == 1):
if (r_t == PING and length > 125):
raise RuntimeError("Ping packet is too large")
r_m = (True if b & 0x80 == 128 else False)
l = b & 0x7F
if (l <= 125):
r_l = l
if (r_m is True):
r_ml = bytearray()
r_s = 4
else:
if (r_l <= 0):
ok = True
else:
r_dt = bytearray()
r_s = 5
elif (l == 126):
r_ll = bytearray()
r_s = 2
else:
r_ll = bytearray()
r_s = 3
elif r_s == 2:
r_ll.append(b)
if (len(r_ll) > 2):
raise RuntimeError(
"Short length exceeded allowable size")
if (len(r_ll) == 2):
r_l = struct.unpack_from("!H", r_ll)[0]
if (r_m is True):
r_ml = bytearray()
r_s = 4
else:
if (r_l <= 0):
ok = True
else:
r_dt = bytearray()
r_s = 5
elif (r_s == 3):
r_ll.append(b)
if (len(r_ll) > 8):
raise RuntimeError(
"Long length exceeded allowable size")
if (len(r_ll) == 8):
r_l = struct.unpack_from("!Q", r_ll)[0]
if (r_m is True):
r_ml = bytearray()
r_s = 4
else:
if (r_l <= 0):
ok = True
else:
r_dt = bytearray()
r_s = 5
elif (r_s == 4):
r_ml.append(b)
if (len(r_ml) > 4):
raise RuntimeError(
"Mask exceeded allowable size")
if (len(r_ml) == 4):
if (r_l <= 0):
ok = True
else:
r_dt = bytearray()
r_s = 5
elif (r_s == 5):
r_dt.append((b ^ r_ml[r_i % 4] if r_m else b))
if (len(r_dt) >= 2**25):
raise RuntimeError(
"Payload exceeded allowable size")
if (r_i + 1 == r_l):
ok = True
else:
r_i += 1
if (ok):
try:
if (r_t == CLOSE or r_t == STREAM
or r_t == TEXT or r_t == BINARY):
pass
elif (r_t == PONG or r_t == PING):
if (len(r_dt) > 125):
raise RuntimeError(
"Control frame length can't be >125"
)
else:
raise RuntimeError("Unknown opcode")
if (r_t == CLOSE):
s = 1000
m = ""
l = len(r_dt)
if (l == 0):
pass
elif (l >= 2):
s = struct.unpack_from("!H",
r_dt[:2])[0]
m = r_dt[2:]
if (s not in [
1000, 1001, 1002, 1003, 1007,
1008, 1009, 1010, 1011, 3000,
3999, 4000, 4999
]):
s = 1002
if (len(m) > 0):
try:
m = m.decode("utf-8",
errors="strict")
except UnicodeDecodeError:
s = 1002
else:
s = 1002
close(s, m)
break
elif (r_f == 0):
if (r_t != STREAM):
if (r_t == PING or r_t == PONG):
raise RuntimeError(
"Control messages can't be fragmented"
)
r_ft = r_t
r_fs = True
r_fd.reset()
if (r_ft == TEXT):
r_fb = []
utf_str = r_fd.decode(r_dt,
final=False)
if (utf_str):
r_fb.append(utf_str)
else:
r_fb = bytearray()
r_fb.extend(r_dt)
else:
if (r_fs is False):
raise RuntimeError(
"Fragmentation protocol error")
if (r_ft == TEXT):
utf_str = r_fd.decode(r_dt,
final=False)
if (utf_str):
r_fb.append(utf_str)
else:
r_fb.extend(r_dt)
elif (r_t == STREAM):
if (r_fs is False):
raise RuntimeError(
"Fragmentation protocol error")
if (r_ft == TEXT):
utf_str = r_fd.decode(r_dt, final=True)
r_fb.append(utf_str)
r_dt = "".join(r_fb)
else:
r_fb.extend(r_dt)
r_dt = r_fb
rf(r_dt)
r_fd.reset()
r_ft = BINARY
r_fs = False
r_fb = None
elif (r_t == PING):
if (isinstance(r_dt, str)):
r_dt = r_dt.encode("utf-8")
l = len(r_dt)
assert (l <= 125)
o = bytearray([PONG | 0x80, l])
if (l > 0):
o.extend(r_dt)
threading.current_thread()._cs_q.append(
(t, o))
elif (r_t == PONG):
pass
else:
if (r_fs is True):
raise RuntimeError(
"Fragmentation protocol error")
if (r_t == TEXT):
try:
r_dt = r_dt.decode("utf8",
errors="strict")
except Exception as exp:
raise RuntimeError(
"Invalid utf-8 payload")
rf(r_dt)
except Exception as e:
traceback.print_exception(
None, e, e.__traceback__)
r_s = 0
r_dt = bytearray()
except BlockingIOError:
pass
b = False
while (len(threading.current_thread()._cs_q) > 0):
threading.current_thread()._tl.acquire()
(t, dt), threading.current_thread(
)._cs_q = threading.current_thread(
)._cs_q[0], threading.current_thread()._cs_q[1:]
threading.current_thread()._tl.release()
r = None
l = len(dt)
i = 0
while (i < l):
try:
j = cs.send(dt[i:])
if (j == 0):
return
i += j
except socket.error as e:
if (e.errno in [errno.EAGAIN, errno.EWOULDBLOCK]):
r = dt[i:]
break
else:
raise e
if (r is not None):
threading.current_thread()._tl.acquire()
threading.current_thread()._cs_q = [
(t, r)
] + threading.current_thread()._cs_q
threading.current_thread()._tl.release()
break
elif (t == CLOSE):
b = True
break
if (b == True):
break
except Exception as e:
traceback.print_exception(None, e, e.__traceback__)
break
cs.close()
if (r_hs):
try:
df()
except Exception as e:
traceback.print_exception(None, e, e.__traceback__)
def set_rx_encoding(self, encoding, errors='replace'):
"""set encoding for received data"""
self.input_encoding = encoding
self.rx_decoder = codecs.getincrementaldecoder(encoding)(errors)
def posix_shell(chan,channel,log_name=None,width=90,height=40):
from webterminal.asgi import channel_layer
stdout = list()
begin_time = time.time()
last_write_time = {'last_activity_time':begin_time}
command = list()
try:
chan.settimeout(0.0)
while True:
try:
x = u(chan.recv(1024))
if len(x) == 0:
channel_layer.send(channel, {'text': json.dumps(['disconnect',smart_unicode('\r\n*** EOF\r\n')]) })
break
now = time.time()
delay = now - last_write_time['last_activity_time']
last_write_time['last_activity_time'] = now
if x == "exit\r\n" or x == "logout\r\n" or x == 'logout':
chan.close()
else:
print('x',x)
if '\r\n' not in x:
command.append(x)
else:
print('rn',command)
print(re.compile('\[.*@.*\][\$#]').search(''.join(command)))
print('command2',''.join(command))
command = list()
if isinstance(x,unicode):
stdout.append([delay,x])
else:
stdout.append([delay,codecs.getincrementaldecoder('UTF-8')('replace').decode(x)])
if isinstance(x,unicode):
channel_layer.send(channel, {'text': json.dumps(['stdout',x]) })
else:
channel_layer.send(channel, {'text': json.dumps(['stdout',smart_unicode(x)]) })
#send message to monitor group
if log_name:
channel_layer.send_group(u'monitor-{0}'.format(log_name.rsplit('/')[1]), {'text': json.dumps(['stdout',smart_unicode(x)]) })
except socket.timeout:
pass
except Exception,e:
print(traceback.print_exc())
channel_layer.send(channel, {'text': json.dumps(['stdout','A bug find,You can report it to me' + smart_unicode(e)]) })
finally:
attrs = {
"version": 1,
"width": width,#int(subprocess.check_output(['tput', 'cols'])),
"height": height,#int(subprocess.check_output(['tput', 'lines'])),
"duration": round(time.time()- begin_time,6),
"command": os.environ.get('SHELL',None),
'title':None,
"env": {
"TERM": os.environ.get('TERM'),
"SHELL": os.environ.get('SHELL','sh')
},
'stdout':list(map(lambda frame: [round(frame[0], 6), frame[1]], stdout))
}
mkdir_p('/'.join(os.path.join(MEDIA_ROOT,log_name).rsplit('/')[0:-1]))
with open(os.path.join(MEDIA_ROOT,log_name), "a") as f:
f.write(json.dumps(attrs, ensure_ascii=True,cls=CustomeFloatEncoder,indent=2))
audit_log=Log.objects.get(channel=channel,log=log_name.rsplit('/')[-1])
audit_log.is_finished = True
audit_log.end_time = timezone.now()
audit_log.save()
#hand ssh terminal exit
queue = get_redis_instance()
redis_channel = queue.pubsub()
queue.publish(channel, json.dumps(['close']))
def create_subprocess_with_handle(command,
display_handle,
*,
shell=False,
cwd,
**kwargs):
'''Writes subprocess output to a display handle as it comes in, and also
returns a copy of it as a string. Throws if the subprocess returns an
error. Note that cwd is a required keyword-only argument, on theory that
peru should never start child processes "wherever I happen to be running
right now."'''
# We're going to get chunks of bytes from the subprocess, and it's possible
# that one of those chunks ends in the middle of a unicode character. An
# incremental decoder keeps those dangling bytes around until the next
# chunk arrives, so that split characters get decoded properly. Use
# stdout's encoding, but provide a default for the case where stdout has
# been redirected to a StringIO. (This happens in tests.)
encoding = sys.stdout.encoding or 'utf8'
decoder_factory = codecs.getincrementaldecoder(encoding)
decoder = decoder_factory(errors='replace')
output_copy = io.StringIO()
# Display handles are context managers. Entering and exiting the display
# handle lets the display know when the job starts and stops.
with display_handle:
stdin = asyncio.subprocess.DEVNULL
stdout = asyncio.subprocess.PIPE
stderr = asyncio.subprocess.STDOUT
if shell:
proc = yield from asyncio.create_subprocess_shell(command,
stdin=stdin,
stdout=stdout,
stderr=stderr,
cwd=cwd,
**kwargs)
else:
proc = yield from asyncio.create_subprocess_exec(*command,
stdin=stdin,
stdout=stdout,
stderr=stderr,
cwd=cwd,
**kwargs)
# Read all the output from the subprocess as its comes in.
while True:
outputbytes = yield from proc.stdout.read(4096)
if not outputbytes:
break
outputstr = decoder.decode(outputbytes)
outputstr_unified = _unify_newlines(outputstr)
display_handle.write(outputstr_unified)
output_copy.write(outputstr_unified)
returncode = yield from proc.wait()
if returncode != 0:
raise subprocess.CalledProcessError(returncode, command,
output_copy.getvalue())
if hasattr(decoder, 'buffer'):
# The utf8 decoder has this attribute, but some others don't.
assert not decoder.buffer, 'decoder nonempty: ' + repr(decoder.buffer)
return output_copy.getvalue()
async def read_output(self):
# We should use incremental decoder because some kernels may
# send us incomplete UTF-8 byte sequences (e.g., Julia).
decoders = (
codecs.getincrementaldecoder('utf8')(errors='replace'),
codecs.getincrementaldecoder('utf8')(errors='replace'),
)
while True:
try:
msg_type, msg_data = await self.output_stream.read()
# TODO: test if save-load runner states is working
# by printing received messages here
if len(msg_data) > self.max_record_size:
msg_data = msg_data[:self.max_record_size]
try:
if msg_type == b'status':
msgpack.unpackb(msg_data, encoding='utf8')
# TODO: not implemented yet
elif msg_type == b'completion':
# As completion is processed asynchronously
# to the main code execution, we directly
# put the result into a separate queue.
await self.completion_queue.put(msg_data)
elif msg_type == b'service-result':
await self.service_queue.put(msg_data)
elif msg_type == b'stdout':
if self.output_queue is None:
continue
await self.output_queue.put(
ResultRecord(
'stdout',
decoders[0].decode(msg_data),
))
elif msg_type == b'stderr':
if self.output_queue is None:
continue
await self.output_queue.put(
ResultRecord(
'stderr',
decoders[1].decode(msg_data),
))
else:
# Normal outputs should go to the current
# output queue.
if self.output_queue is None:
continue
await self.output_queue.put(
ResultRecord(
msg_type.decode('ascii'),
msg_data.decode('utf8'),
))
except asyncio.QueueFull:
pass
if msg_type == b'build-finished':
# finalize incremental decoder
decoders[0].decode(b'', True)
decoders[1].decode(b'', True)
elif msg_type == b'finished':
# finalize incremental decoder
decoders[0].decode(b'', True)
decoders[1].decode(b'', True)
self.finished_at = time.monotonic()
except (asyncio.CancelledError, aiozmq.ZmqStreamClosed,
GeneratorExit):
break
except Exception:
log.exception('unexpected error')
break
def test_decoder():
def checkauto(encoding,
input="<?xml version='1.0' encoding='x'?>gürk\u20ac"):
# Check stateless decoder
d = codecs.getdecoder("xml")
assert d(input.encode(encoding))[0] == input.replace(
"'x'", repr(encoding))
# Check stateless decoder with specified encoding
assert d(input.encode(encoding),
encoding=encoding)[0] == input.replace("'x'", repr(encoding))
# Check incremental decoder
id = codecs.getincrementaldecoder("xml")()
assert "".join(
id.iterdecode(bytes([c])
for c in input.encode(encoding))) == input.replace(
"'x'", repr(encoding))
# Check incremental decoder with specified encoding
id = codecs.getincrementaldecoder("xml")(encoding)
assert "".join(
id.iterdecode(bytes([c])
for c in input.encode(encoding))) == input.replace(
"'x'", repr(encoding))
id = codecs.getincrementaldecoder("xml")(encoding="ascii")
assert id.decode(b"<?xml version='1.0' encoding='utf-16'?>"
) == "<?xml version='1.0' encoding='ascii'?>"
# Autodetectable encodings
checkauto("utf-8-sig")
checkauto("utf-16")
checkauto("utf-16-le")
checkauto("utf-16-be")
if haveutf32:
checkauto("utf-32")
checkauto("utf-32-le")
checkauto("utf-32-be")
def checkdecl(
encoding,
input="<?xml version='1.0' encoding={encoding!r}?><gürk>\u20ac</gürk>"
):
# Check stateless decoder with encoding autodetection
d = codecs.getdecoder("xml")
input = input.format(encoding=encoding)
assert d(input.encode(encoding))[0] == input
# Check stateless decoder with specified encoding
assert d(input.encode(encoding), encoding=encoding)[0] == input
# Check incremental decoder with encoding autodetection
id = codecs.getincrementaldecoder("xml")()
assert "".join(
id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input
# Check incremental decoder with specified encoding
id = codecs.getincrementaldecoder("xml")(encoding)
assert "".join(
id.iterdecode(bytes([c]) for c in input.encode(encoding))) == input
# Use correct declaration
checkdecl("utf-8")
checkdecl("iso-8859-1",
"<?xml version='1.0' encoding={encoding!r}?><gürk/>")
checkdecl("iso-8859-15")
checkdecl("cp1252")
# No recursion
with pytest.raises(ValueError):
b"<?xml version='1.0' encoding='xml'?><gurk/>".decode("xml")
def run_attach(cmd, output_cb=None):
""" run command and attach output to cb
Arguments:
cmd: shell command
output_cb: where to display output
"""
stdoutmaster, stdoutslave = pty.openpty()
subproc = Popen(cmd, shell=True,
stdout=stdoutslave,
stderr=PIPE)
os.close(stdoutslave)
decoder = codecs.getincrementaldecoder('utf-8')()
def last_ten_lines(s):
chunk = s[-1500:]
lines = chunk.splitlines(True)
return ''.join(lines[-10:]).replace('\r', '')
decoded_output = ""
try:
while subproc.poll() is None:
try:
b = os.read(stdoutmaster, 512)
except OSError as e:
if e.errno != errno.EIO:
raise
break
else:
final = False
if not b:
final = True
decoded_chars = decoder.decode(b, final)
if decoded_chars is None:
continue
decoded_output += decoded_chars
if output_cb:
ls = last_ten_lines(decoded_output)
output_cb(ls)
if final:
break
finally:
os.close(stdoutmaster)
if subproc.poll() is None:
subproc.kill()
subproc.wait()
errors = [l.decode('utf-8') for l in subproc.stderr.readlines()]
if output_cb:
output_cb(last_ten_lines(decoded_output))
errors = ''.join(errors)
if subproc.returncode == 0:
return decoded_output.strip()
else:
raise Exception("Problem running {0} "
"{1}:{2}".format(cmd,
subproc.returncode))
def test_latex_incremental_decoder_setstate():
encoder = codecs.getincrementaldecoder('latex')()
state = (u'', 0)
with pytest.raises(NotImplementedError):
encoder.setstate(state)
def set_rx_encoding(self, encoding, errors='replace'):
self.input_encoding = encoding
self.rx_decoder = codecs.getincrementaldecoder(encoding)(errors)
def __init__(self, timeout=60, maxread=2000, searchwindowsize=None,
logfile=None, encoding=None, codec_errors='strict'):
self.stdin = sys.stdin
self.stdout = sys.stdout
self.stderr = sys.stderr
self.searcher = None
self.ignorecase = False
self.before = None
self.after = None
self.match = None
self.match_index = None
self.terminated = True
self.exitstatus = None
self.signalstatus = None
# status returned by os.waitpid
self.status = None
# the child file descriptor is initially closed
self.child_fd = -1
self.timeout = timeout
self.delimiter = EOF
self.logfile = logfile
# input from child (read_nonblocking)
self.logfile_read = None
# output to send (send, sendline)
self.logfile_send = None
# max bytes to read at one time into buffer
self.maxread = maxread
# Data before searchwindowsize point is preserved, but not searched.
self.searchwindowsize = searchwindowsize
# Delay used before sending data to child. Time in seconds.
# Set this to None to skip the time.sleep() call completely.
self.delaybeforesend = 0.05
# Used by close() to give kernel time to update process status.
# Time in seconds.
self.delayafterclose = 0.1
# Used by terminate() to give kernel time to update process status.
# Time in seconds.
self.delayafterterminate = 0.1
# Delay in seconds to sleep after each call to read_nonblocking().
# Set this to None to skip the time.sleep() call completely: that
# would restore the behavior from pexpect-2.0 (for performance
# reasons or because you don't want to release Python's global
# interpreter lock).
self.delayafterread = 0.0001
self.softspace = False
self.name = '<' + repr(self) + '>'
self.closed = True
# Unicode interface
self.encoding = encoding
self.codec_errors = codec_errors
if encoding is None:
# bytes mode (accepts some unicode for backwards compatibility)
self._encoder = self._decoder = _NullCoder()
self.string_type = bytes
self.buffer_type = BytesIO
self.crlf = b'\r\n'
if PY3:
self.allowed_string_types = (bytes, str)
self.linesep = os.linesep.encode('ascii')
def write_to_stdout(b):
try:
return sys.stdout.buffer.write(b)
except AttributeError:
# If stdout has been replaced, it may not have .buffer
return sys.stdout.write(b.decode('ascii', 'replace'))
self.write_to_stdout = write_to_stdout
else:
self.allowed_string_types = (basestring,) # analysis:ignore
self.linesep = os.linesep
self.write_to_stdout = sys.stdout.write
else:
# unicode mode
self._encoder = codecs.getincrementalencoder(encoding)(codec_errors)
self._decoder = codecs.getincrementaldecoder(encoding)(codec_errors)
self.string_type = text_type
self.buffer_type = StringIO
self.crlf = u'\r\n'
self.allowed_string_types = (text_type, )
if PY3:
self.linesep = os.linesep
else:
self.linesep = os.linesep.decode('ascii')
# This can handle unicode in both Python 2 and 3
self.write_to_stdout = sys.stdout.write
# storage for async transport
self.async_pw_transport = None
# This is the read buffer. See maxread.
self._buffer = self.buffer_type()
class EmailHandler:
decoder = codecs.getincrementaldecoder('utf-8')()
def __init__(self, login, password):
self.mail = imaplib.IMAP4_SSL('imap.gmail.com', 993)
self.mail.login(login, password)
self.mail.list()
self.mail.select("inbox")
self.c = 0
typ, data = self.mail.search(None, 'All')
all_id = data[0].split()
result, data = self.mail.fetch(all_id[-10], "(RFC822)")
raw_email = data[0][1]
self.my_mail = email.message_from_bytes(raw_email)
def get_last_mails(self, last_mails: int):
self.mail.list()
self.mail.select("inbox")
self.c = 0
typ, data = self.mail.search(None, 'All')
all_id = data[0].split()
return_value = []
for id in all_id[-1 * last_mails:]:
result, data = self.mail.fetch(id, "(RFC822)")
raw_email = data[0][1]
self.my_mail = email.message_from_bytes(raw_email)
return_value.append(self.my_mail)
return_value.reverse()
return return_value
def get_text(self):
my_mail = self.my_mail
while my_mail.is_multipart():
my_mail = my_mail.get_payload(0)
content = my_mail.get_payload(decode=True)
return EmailHandler.decoder.decode(content)
def get_from(self):
my_mail = self.my_mail
decode1 = decode_header(my_mail['From'])[0]
res1 = decode1[0]
if decode1[1] is not None:
temp = codecs.getincrementaldecoder(decode1[1])()
res1 = temp.decode(decode1[0])
if len(decode_header(my_mail['From'])) == 2:
decode2 = decode_header(my_mail['From'])[1]
res2 = decode2[0]
if decode2[1] is not None:
temp = codecs.getincrementaldecoder(decode2[1])()
res2 = temp.decode(decode2[0])
return [res1, res2]
return ["", res1]
def get_subject(self):
my_mail = self.my_mail
subject = my_mail['Subject'].split()
result = ""
for temp in subject:
decode1 = decode_header(temp)[0]
res = decode1[0]
if decode1[1] is not None:
temp = codecs.getincrementaldecoder(decode1[1])()
res = temp.decode(decode1[0])
result += res
else:
result += res + " "
return result
def download_attachment(self):
email_message = self.my_mail
c = 0
k = 1
for part in email_message.walk():
if part.get_content_maintype() == 'multipart':
continue
if part.get('Content-Disposition') is None:
continue
file_name = part.get_filename()
if file_name is not None:
names = file_name.split()
file_name: str = ""
for name in names:
decode1 = decode_header(name)[0]
part_name = decode1[0]
if decode1[1] is not None:
temp = codecs.getincrementaldecoder(decode1[1])()
part_name = temp.decode(decode1[0])
file_name += part_name
else:
file_name += part_name + " "
if bool(file_name):
file_path = os.path.join('/home/ubuntu/Daniil/Gmail/Files',
file_name)
if not os.path.isfile(file_path):
fp = open(file_path, 'wb')
fp.write(part.get_payload(decode=True))
fp.close()
c += 1
def __call__(self, *args, **kwargs):
if platform.system() == 'Windows':
self.func(*args, **kwargs)
return
pid_testrunner = os.getpid()
pid, master_fd = pty.fork()
if pid == self._CHILD_PID:
# child process executes function, raises exception
# if failed, causing a non-zero exit code, using the
# protected _exit() function of ``os``; to prevent the
# 'SystemExit' exception from being thrown.
cov = init_subproc_coverage(
"@as_subprocess-{pid};{func_name}(*{args}, **{kwargs})"
.format(pid=os.getpid(), func_name=self.func,
args=args, kwargs=kwargs))
try:
self.func(*args, **kwargs)
except Exception:
e_type, e_value, e_tb = sys.exc_info()
o_err = list()
for line in traceback.format_tb(e_tb):
o_err.append(line.rstrip().encode('utf-8'))
o_err.append(('-=' * 20).encode('ascii'))
o_err.extend([_exc.rstrip().encode('utf-8') for _exc in
traceback.format_exception_only(
e_type, e_value)])
os.write(sys.__stdout__.fileno(), b'\n'.join(o_err))
os.close(sys.__stdout__.fileno())
os.close(sys.__stderr__.fileno())
os.close(sys.__stdin__.fileno())
if cov is not None:
cov.stop()
cov.save()
os._exit(1)
else:
if cov is not None:
cov.stop()
cov.save()
os._exit(0)
# detect rare fork in test runner, when bad bugs happen
if pid_testrunner != os.getpid():
print('TEST RUNNER HAS FORKED, {0}=>{1}: EXIT'
.format(pid_testrunner, os.getpid()), file=sys.stderr)
os._exit(1)
exc_output = six.text_type()
decoder = codecs.getincrementaldecoder(self.encoding)()
while True:
try:
_exc = os.read(master_fd, 65534)
except OSError:
# linux EOF
break
if not _exc:
# bsd EOF
break
exc_output += decoder.decode(_exc)
# parent process asserts exit code is 0, causing test
# to fail if child process raised an exception/assertion
pid, status = os.waitpid(pid, 0)
os.close(master_fd)
# Display any output written by child process
# (esp. any AssertionError exceptions written to stderr).
exc_output_msg = 'Output in child process:\n%s\n%s\n%s' % (
u'=' * 40, exc_output, u'=' * 40,)
assert exc_output == '', exc_output_msg
# Also test exit status is non-zero
assert os.WEXITSTATUS(status) == 0
def __init__(self, pid, fd, encoding='utf-8', codec_errors='strict'):
super(SimplePty, self).__init__(pid, fd)
self.encoding = encoding
self.codec_errors = codec_errors
self.decoder = codecs.getincrementaldecoder(encoding)(
errors=codec_errors)
async def read_message(self) -> Optional[Data]:
"""
Read a single message from the connection.
Re-assemble data frames if the message is fragmented.
Return ``None`` when the closing handshake is started.
"""
frame = await self.read_data_frame(max_size=self.max_size)
# A close frame was received.
if frame is None:
return None
if frame.opcode == OP_TEXT:
text = True
elif frame.opcode == OP_BINARY:
text = False
else: # frame.opcode == OP_CONT
raise WebSocketProtocolError("Unexpected opcode")
# Shortcut for the common case - no fragmentation
if frame.fin:
return frame.data.decode("utf-8") if text else frame.data
# 5.4. Fragmentation
chunks: List[Data] = []
max_size = self.max_size
if text:
decoder_factory = codecs.getincrementaldecoder("utf-8")
# https://github.com/python/typeshed/pull/2752
decoder = decoder_factory(errors="strict") # type: ignore
if max_size is None:
def append(frame: Frame) -> None:
nonlocal chunks
chunks.append(decoder.decode(frame.data, frame.fin))
else:
def append(frame: Frame) -> None:
nonlocal chunks, max_size
chunks.append(decoder.decode(frame.data, frame.fin))
max_size -= len(frame.data)
else:
if max_size is None:
def append(frame: Frame) -> None:
nonlocal chunks
chunks.append(frame.data)
else:
def append(frame: Frame) -> None:
nonlocal chunks, max_size
chunks.append(frame.data)
max_size -= len(frame.data)
append(frame)
while not frame.fin:
frame = await self.read_data_frame(max_size=max_size)
if frame is None:
raise WebSocketProtocolError("Incomplete fragmented message")
if frame.opcode != OP_CONT:
raise WebSocketProtocolError("Unexpected opcode")
append(frame)
# mypy cannot figure out that chunks have the proper type.
return ("" if text else b"").join(chunks) # type: ignore
def __init__(self, generator, encoding):
self._gen = generator.__aiter__()
self._decoder = codecs.getincrementaldecoder(encoding)(errors='ignore')
self._flag = False
def __post_init__(self, encoding: str, errors: str) -> None:
decoder_class = codecs.getincrementaldecoder(encoding)
self._decoder = decoder_class(errors=errors)
def __init__(self, encoding, errors, textmode=False):
if encoding and not textmode:
self._decoder = codecs.getincrementaldecoder(encoding)(errors)
else:
self._decoder = None
def read_message(self):
"""
Read a single message from the connection.
Re-assemble data frames if the message is fragmented.
Return ``None`` when the closing handshake is started.
"""
frame = yield from self.read_data_frame(max_size=self.max_size)
# A close frame was received.
if frame is None:
return
if frame.opcode == OP_TEXT:
text = True
elif frame.opcode == OP_BINARY:
text = False
else: # frame.opcode == OP_CONT
raise WebSocketProtocolError("Unexpected opcode")
# Shortcut for the common case - no fragmentation
if frame.fin:
return frame.data.decode('utf-8') if text else frame.data
# 5.4. Fragmentation
chunks = []
max_size = self.max_size
if text:
decoder = codecs.getincrementaldecoder('utf-8')(errors='strict')
if max_size is None:
def append(frame):
nonlocal chunks
chunks.append(decoder.decode(frame.data, frame.fin))
else:
def append(frame):
nonlocal chunks, max_size
chunks.append(decoder.decode(frame.data, frame.fin))
max_size -= len(frame.data)
else:
if max_size is None:
def append(frame):
nonlocal chunks
chunks.append(frame.data)
else:
def append(frame):
nonlocal chunks, max_size
chunks.append(frame.data)
max_size -= len(frame.data)
append(frame)
while not frame.fin:
frame = yield from self.read_data_frame(max_size=max_size)
if frame is None:
raise WebSocketProtocolError("Incomplete fragmented message")
if frame.opcode != OP_CONT:
raise WebSocketProtocolError("Unexpected opcode")
append(frame)
return ('' if text else b'').join(chunks)
def decode(self, text_utf8, text_latex, inputenc=None):
encoding = 'latex+' + inputenc if inputenc else 'latex'
decoder = codecs.getincrementaldecoder(encoding)()
decoded_parts = (decoder.decode(text_latex_part, final)
for text_latex_part, final in split_input(text_latex))
self.assertEqual(text_utf8, u''.join(decoded_parts))
"""asciicast records
This module provides functions and classes to manipulate asciicast records. Both v1 and v2
format are supported for decoding. For encoding, only the v2 format is available. The
specification of the two formats are available here:
[1] https://github.com/asciinema/asciinema/blob/develop/doc/asciicast-v1.md
[2] https://github.com/asciinema/asciinema/blob/develop/doc/asciicast-v2.md
"""
import abc
import codecs
import json
from collections import namedtuple
from typing import Generator, Iterable, Union
utf8_decoder = codecs.getincrementaldecoder('utf-8')('replace')
class AsciiCastError(Exception):
pass
class AsciiCastV2Record(abc.ABC):
"""Generic Asciicast v2 record format"""
@abc.abstractmethod
def to_json_line(self):
raise NotImplementedError
@classmethod
def from_json_line(cls, line):
"""Raise AsciiCastError if line is not a valid asciicast v2 record"""
try:
def test_latex_incremental_decoder_setstate():
encoder = codecs.getincrementaldecoder('latex')()
state = (u'', 0)
nose.tools.assert_raises(NotImplementedError,
lambda: encoder.setstate(state))
def encode(buf):
"Null encode"
return buf
if Python3:
def decode(buf):
"Decode buffer for Python 3"
return buf.decode(Encoding)
def encode(buf):
"Encode buffer for Python 3"
return buf.encode(Encoding)
getincrementaldecoder = codecs.getincrementaldecoder(Encoding)
else:
decode, encode = NullCodec.decode, NullCodec.encode
def getincrementaldecoder():
"Return null codec for Python 2"
return NullCodec
try:
# pylint: disable=import-error
oldpexpect = None
import pexpect as oldpexpect
# pylint: enable=import-error
class Pexpect(object):
def _multiplex(ep,
read_fds,
callback_raw,
callback_linebuffered,
encoding='utf-8',
write=None,
timeout=1,
buffer_size=80):
# Register the file descriptors (stdout + stderr) with the epoll object
# so that we'll get notifications when data are ready to read
for fd in read_fds:
ep.register(fd, POLL_IN | POLL_PRI)
# Register a write file descriptor
if write:
ep.register(write[0], POLL_OUT)
# Offset into the `write[1]` buffer where we should continue writing to stdin
offset = 0
# We need to keep track of which file descriptors have already been drained
# because when running under `pytest` it seems that all `epoll` events are
# received twice so using solely `ep.unregister(fd)` will not work
hupped = set()
# Total number of 'hupped' file descriptors we expect
num_expected = len(read_fds) + (1 if write else 0)
# Set up file-descriptor specific buffers where we'll buffer the output
buf = {fd: bytes() for fd in read_fds}
if encoding:
linebufs = {fd: '' for fd in read_fds}
decoders = {
fd: codecs.getincrementaldecoder(encoding)()
for fd in read_fds
}
def _get_fd_type(fd):
"""
File descriptors passed via `read_fds` are always representing [stdout, stderr],
since arrays start at index 0, we need to add 1 to get the real symbolic value
`STDOUT` or `STDERR`.
"""
return read_fds.index(fd) + 1
while not ep.closed and len(hupped) != num_expected:
events = ep.poll(timeout)
for fd, event in events:
if event == POLL_HUP:
hupped.add(fd)
ep.unregister(fd)
if event & (POLL_IN | POLL_PRI) != 0:
fd_type = _get_fd_type(fd)
read = os.read(fd, buffer_size)
callback_raw((fd, fd_type), read)
if encoding:
linebufs[fd] += decoders[fd].decode(read)
while '\n' in linebufs[fd]:
pre, post = linebufs[fd].split('\n', 1)
linebufs[fd] = post
callback_linebuffered((fd, fd_type), pre)
buf[fd] += read
elif event == POLL_OUT:
# Write data to pipe, `os.write` returns the number of bytes written,
# thus we need to offset
wfd, data = write
if fd in hupped:
continue
offset += os.write(fd, data[offset:])
if offset == len(data):
hupped.add(fd)
ep.unregister(fd)
os.close(fd)
# Process leftovers from line buffering
if encoding:
for (fd, lb) in linebufs.items():
if lb:
# [stdout, stderr] is relayed, stdout=1 a stderr=2
# as the field starting indexed is 0, so the +1 needs to be added
callback_linebuffered((fd, _get_fd_type(fd)), lb)
return buf
def test_utf8sig(self):
import codecs
d = codecs.getincrementaldecoder("utf-8-sig")()
s = "spam"
assert d.decode(s.encode("utf-8-sig")) == s
