def _build_dchunks(self):
ddata = []
directory = list(self._directory)
directory.sort(cmp=lambda x, y:
cmp(x.name.lower(), y.name.lower()))
qrn = 1 + (1 << 2)
dchunk = StringIO()
dcount = 0
quickref = []
name = directory[0].name
for entry in directory:
en = entry.name.encode('utf-8') if entry.name else entry.name
next = ''.join([decint(len(en)), en,
decint(entry.section), decint(entry.offset),
decint(entry.size)])
usedlen = dchunk.tell() + len(next) + (len(quickref) * 2) + 52
if usedlen >= DCHUNK_SIZE:
ddata.append((dchunk.getvalue(), quickref, dcount, name))
dchunk = StringIO()
dcount = 0
quickref = []
name = en
if (dcount % qrn) == 0:
quickref.append(dchunk.tell())
dchunk.write(next)
dcount = dcount + 1
ddata.append((dchunk.getvalue(), quickref, dcount, name))
cidmax = len(ddata) - 1
rdcount = 0
dchunks = []
dcounts = []
ichunk = None
if len(ddata) > 1:
ichunk = StringIO()
for cid, (content, quickref, dcount, name) in izip(count(), ddata):
dchunk = StringIO()
prev = cid - 1 if cid > 0 else ULL_NEG1
next = cid + 1 if cid < cidmax else ULL_NEG1
rem = DCHUNK_SIZE - (len(content) + 50)
pad = rem - (len(quickref) * 2)
dchunk.write('AOLL')
dchunk.write(pack('<IQQQQQ', rem, cid, prev, next, rdcount, 1))
dchunk.write(content)
dchunk.write('\0' * pad)
for ref in reversed(quickref):
dchunk.write(pack('<H', ref))
dchunk.write(pack('<H', dcount))
rdcount = rdcount + dcount
dchunks.append(dchunk.getvalue())
dcounts.append(dcount)
if ichunk:
ichunk.write(decint(len(name)))
ichunk.write(name)
ichunk.write(decint(cid))
if ichunk:
rem = DCHUNK_SIZE - (ichunk.tell() + 16)
pad = rem - 2
ichunk = ''.join(['AOLI', pack('<IQ', rem, len(dchunks)),
ichunk.getvalue(), ('\0' * pad), pack('<H', len(dchunks))])
return dcounts, dchunks, ichunk
def export_msg(fn):
base_tbl = GetTable()
base = open(fn , "rb")
fp = StringIO()
fp.write(base.read())
fp.seek(0)
cnlines = codecs.open("t3b_cn.txt" , "rb" , "utf-16").readlines()
index_end_offset = struct.unpack("I" , fp.read(4))[0]
string_list = makestr(cnlines)
nums = len(string_list)
fp.seek(0x18c48)
fp.truncate()
tmp = fp.tell()
for i in xrange(nums):
fp.seek(0,2)
tmp = fp.tell()
string = string2hex(string_list[i] , base_tbl)
fp.write(string)
fp.seek(0x5dfc + i*0xc)
fp.write(struct.pack("I" , tmp))
fp.write(struct.pack("I" , len(string)/2))
fp.seek(0,2)
data = fp.getvalue()
dest = open("import\\00000019.lang" , "wb")
dest.write(data)
dest.close()
def read_framed_message(self):
"""Read the response from the wire and return it."""
message = StringIO()
while True:
buff = StringIO()
buffer_header = self.sock.recv(avro.ipc.BUFFER_HEADER_LENGTH)
buffer_length = \
avro.ipc.BIG_ENDIAN_INT_STRUCT.unpack(buffer_header)[0]
if buffer_length == "":
raise avro.ipc.ConnectionClosedException(
"Socket read 0 bytes."
)
elif buffer_length == 0:
# A 0-length buffer indicates the end of the message
return message.getvalue()
while buff.tell() < buffer_length:
chunk = self.sock.recv(buffer_length - buff.tell())
if chunk == "":
raise avro.ipc.ConnectionClosedException(
"Socket read 0 bytes."
)
buff.write(chunk)
message.write(buff.getvalue())
def post(self, instance):
instance = list(instance)
name = instance[0].application + '-' + \
instance[0].msgtime.strftime('%Y%m%d%H%M%S')
from cStringIO import StringIO
s = StringIO()
count = 1
for msg in instance:
if self.size:
loc = s.tell()
msg.toFile(s)
if self.size and (s.tell() > self.size):
s.seek(loc)
s.truncate()
url = self._post(name+'p'+count, s.getvalue())
print "{0:<40} : {1}".format(name+'p'+count, url)
count += 1
s.seek(0)
s.truncate()
msg.toFile(s)
if count > 1:
name += 'p'+count
url = self._post(name, s.getvalue())
print "{0:<40} : {1}".format(name, url)
def read(self, _len_to_read):
buf = StringIO()
try:
while buf.tell() < _len_to_read:
tmp = self._socket.recv( _len_to_read - buf.tell() )
buf.write( tmp )
if is_debug_verbose():
import hexdump
print( "\n -------------------\n" )
print( " To read was: " + str( _len_to_read ) )
print( hexdump.hexdump(tmp, 'return') )
print( " left: " + str( _len_to_read - buf.tell() ) )
buf.seek(0)
return buf.read( _len_to_read )
except socket.timeout as e:
# we don't set false because of
# RecordUpdateMessage/RecordCreateMessage trick
""":see RecordUpdateMessage.py """
raise e
except Exception as e:
self._connected = False
raise e
finally:
buf.close()
def dlexpect(self, pattern, flags=0, deadline=None):
"""Loop recv listening for a provided regular expression."""
# pull default timeout
if deadline is None:
deadline = self._deadline
if deadline < 1000:
deadline += time()
buff = StringIO()
# loop until pattern has been found
while not pattern.search(buff.getvalue()):
# wait for data on the socket
t = time()
timeout = (deadline-t) if (deadline-t>0) else 0.0
if len(select([self],[],[], timeout)[0]) == 0:
# deadline reached, terminate
return ''
# append response to buffer
p = buff.tell()
try:
buff.write(self.recv(100, flags))
except socket.error, e:
raise MythError(MythError.SOCKET, e.args)
if buff.tell() == p:
# no data read from a 'ready' socket, connection terminated
raise MythError(MythError.CLOSEDSOCKET)
if timeout == 0:
break
def dlrecv(self, bufsize, flags=0, deadline=None):
# pull default timeout
if deadline is None:
deadline = self._deadline
if deadline < 1000:
deadline += time()
buff = StringIO()
# loop until necessary data has been received
while bufsize > buff.tell():
# wait for data on the socket
t = time()
timeout = (deadline-t) if (deadline-t>0) else 0.0
if len(select([self],[],[], timeout)[0]) == 0:
# deadline reached, terminate
return u''
# append response to buffer
p = buff.tell()
try:
buff.write(self.recv(bufsize-buff.tell(), flags))
except socket.error, e:
raise MythError(MythError.SOCKET, e.args)
if buff.tell() == p:
# no data read from a 'ready' socket, connection terminated
raise MythError(MythError.SOCKET, (54, 'Connection reset by peer'))
if timeout == 0:
break
class Section(object):
def __init__(self, name):
self.name = name
self.content = StringIO()
# Fake-out pre-relocation symbol sizes
self.base = 0x10000
def offset():
def fget(self):
return self.content.tell()
def fset(self, value):
self.content.seek(value)
return property(fget=fget, fset=fset)
offset = offset()
def __len__(self):
offset = self.content.tell()
try:
self.content.seek(0, 2)
result = self.content.tell()
finally:
self.content.seek(offset, 0)
return result
def eval(self):
return set([self]), self.base
def __getattr__(self, name):
return getattr(self.content, name)
def __repr__(self):
return "Section(%r)" % (self.name,)
def send(self, header, message):
""" Sends a header/msg/footer out the socket. Takes care of computing
length field for header and checksum field for footer. """
msg_buff = StringIO()
message.translator().preserialize()
message.translator().serialize(msg_buff)
pad_count = -msg_buff.tell() % 4
msg_buff.write("\x00" * pad_count)
footer = msg.CommonFooter(end=msg.CommonFooter.END)
header.length = msg_buff.tell() + footer.translator().size
# Write header and message to main buffer.
buff = StringIO()
header.translator().serialize(buff)
buff.write(msg_buff.getvalue())
# Write footer.
footer_start = buff.tell()
footer.translator().serialize(buff)
# Compute checksum.
buff.seek(0)
footer.checksum = 65536 - self._checksum(buff)
# Rewrite footer with correct checksum.
buff.seek(footer_start)
footer.translator().serialize(buff)
self.sock.send(buff.getvalue())
def camara_stream():
global capture
capture = cv2.VideoCapture(0)
capture.set(cv2.cv.CV_CAP_PROP_FRAME_WIDTH, 320)
capture.set(cv2.cv.CV_CAP_PROP_FRAME_HEIGHT, 240)
capture.set(cv2.cv.CV_CAP_PROP_SATURATION, 0.2)
try:
while True:
rc, img = capture.read()
if not rc:
continue
imgRGB = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
jpg = Image.fromarray(imgRGB)
tmpfp = StringIO()
jpg.save(tmpfp, 'JPEG')
tmpfp.seek(0, os.SEEK_END)
length = tmpfp.tell()
tmpfp.seek(0)
yield '--jpgboundary\r\n'
yield 'Content-length: image/jpeg\r\n'
yield 'Content-length: %d\r\n' % tmpfp.tell()
yield '\r\n'
yield '%s\r\n\r\n' % tmpfp.getvalue()
del tmpfp # Performance?
time.sleep(RETARDO)
except Exception as e:
print e
finally:
capture.release()
def do_http(method, url, body=""):
if isinstance(body, str):
body = StringIO(body)
elif isinstance(body, unicode):
raise TypeError("do_http body must be a bytestring, not unicode")
else:
# We must give a Content-Length header to twisted.web, otherwise it
# seems to get a zero-length file. I suspect that "chunked-encoding"
# may fix this.
assert body.tell
assert body.seek
assert body.read
scheme, host, port, path = parse_url(url)
if scheme == "http":
c = httplib.HTTPConnection(host, port)
elif scheme == "https":
c = httplib.HTTPSConnection(host, port)
else:
raise ValueError("unknown scheme '%s', need http or https" % scheme)
c.putrequest(method, path)
c.putheader("Hostname", host)
c.putheader("User-Agent", allmydata.__full_version__ + " (tahoe-client)")
c.putheader("Accept", "text/plain, application/octet-stream")
c.putheader("Connection", "close")
old = body.tell()
body.seek(0, os.SEEK_END)
length = body.tell()
body.seek(old)
c.putheader("Content-Length", str(length))
try:
c.endheaders()
except socket_error, err:
return BadResponse(url, err)
def build_cmap(string):
length = len(string)
buffer = StringIO()
buffer.seek(0)
base_stream = Base(buffer)
buffer.write("CMAP")
base_stream.WriteBEUInt32(length * 4 + 18)
base_stream.WriteBEUInt16(0)
base_stream.WriteBEUInt16(0XFFFF)
base_stream.WriteBEUInt16(2)
base_stream.WriteBEUInt16(0)
base_stream.WriteBEUInt32(0)
base_stream.WriteBEUInt16(length)
for i in xrange(length):
char = string[i]
uchar = struct.unpack("H" ,char.encode("utf-16")[2:])[0]
base_stream.WriteBEUInt16(uchar)
base_stream.WriteBEUInt16(i)
buffer.seek(0,2)
end_pos = buffer.tell()
print(hex(end_pos))
if end_pos % 4 != 0:
buffer.write("\x00" * (0x4 - end_pos%4))
buffer.seek(0,2)
end_pos = buffer.tell()
buffer.seek(4)
base_stream.WriteBEUInt32(end_pos)
cmapdata = buffer.getvalue()
return cmapdata
def build_block(base_offset , string_list):
buffer = StringIO()
buffer.write("TXT2")
buffer.write("\x00\x00\x00\x00" * 3)
buffer.write(struct.pack(">I" , len(string_list)))
buffer.write("\x00\x00\x00\x00" * len(string_list))
tmp = buffer.tell()
for i in xrange(len(string_list)):
buffer.seek(tmp)
boffset = buffer.tell()
string = string_list[i]
bin = string2hex(string)
buffer.write(bin)
tmp = buffer.tell()
buffer.seek(0x14 + i * 4)
buffer.write(struct.pack(">I" , boffset - 0x10))
buffer.seek(0,2)
end = buffer.tell()
buffer.seek(4)
buffer.write(struct.pack(">I" , end - 0x10))
data = buffer.getvalue()
if not len(data)%0x10 == 0:
data += ("\xAB" * (0x10 - len(data)%0x10))
return data
class AutoTemporaryFile(object):
"""Initially a StringIO, but becomes a TemporaryFile if it grows large.
Not thread safe.
"""
def __init__(self, threshold=10*1024*1024):
self._threshold = threshold
self._f = StringIO()
def read(self, n=None):
if n is not None:
return self._f.read(n)
else:
return self._f.read()
def seek(self, pos, mode=0):
self._f.seek(pos, mode)
def tell(self):
return self._f.tell()
def close(self):
self._f.close()
def write(self, data):
threshold = self._threshold
if threshold and self._f.tell() + len(data) >= threshold:
# convert to TemporaryFile
self._threshold = 0
f = tempfile.TemporaryFile()
f.write(self._f.getvalue())
f.seek(self._f.tell())
self._f = f
self._f.write(data)
def test_decodeWithCompression(self):
"""
If the leading byte of an encoded label (in bytes read from a stream
passed to L{Name.decode}) has its two high bits set, the next byte is
treated as a pointer to another label in the stream and that label is
included in the name being decoded.
"""
# Slightly modified version of the example from RFC 1035, section 4.1.4.
stream = StringIO(
"x" * 20 +
"\x01f\x03isi\x04arpa\x00"
"\x03foo\xc0\x14"
"\x03bar\xc0\x20")
stream.seek(20)
name = dns.Name()
name.decode(stream)
# Verify we found the first name in the stream and that the stream
# position is left at the first byte after the decoded name.
self.assertEqual("f.isi.arpa", name.name)
self.assertEqual(32, stream.tell())
# Get the second name from the stream and make the same assertions.
name.decode(stream)
self.assertEqual(name.name, "foo.f.isi.arpa")
self.assertEqual(38, stream.tell())
# Get the third and final name
name.decode(stream)
self.assertEqual(name.name, "bar.foo.f.isi.arpa")
self.assertEqual(44, stream.tell())
def read(self, nbytes=None):
"""
Read up to specified amount from buffer, or whatever is available.
"""
# flush existing buffer
self._rollback_pool = []
data = StringIO()
while True:
try:
# get first item, or return if no more blocks are avaialable
tmp = self._buffer[0]
except IndexError:
break
if nbytes:
# read only what is requested
data.write(tmp.read(nbytes-data.tell()))
else:
# read all that is available
data.write(tmp.read(tmp.blocksize))
if tmp.EOF:
# block is exhausted, cycle it into the rollback pool
self._rollback_pool.append(self._buffer.popleft())
else:
# end of data or request reached, return
break
self._nbytes += data.tell()
return data.getvalue()
class Stream:
"""Threadsafe string stream"""
def __init__(self):
self.stream = StringIO()
self.lock = Lock()
self.position = 0
def read(self, n=-1):
"""Read to the front of the stream"""
with self.lock:
prev_position = self.stream.tell()
self.stream.seek(self.position)
result = self.stream.read(n)
self.position = self.stream.tell()
self.stream.seek(prev_position)
return result
def write(self, s):
"""Write to the back of the stream"""
with self.lock:
prev_position = self.stream.tell()
self.stream.seek(0, os.SEEK_END)
self.stream.write(s)
self.stream.seek(prev_position)
def __getattr__(self, attr):
return getattr(self.stream, attr)
def DecodeWay(self):
length = Encoding.DecodeVarint(self.handle)
objData = self.handle.read(length)
#Decode object ID
objDataStream = BytesIO(objData)
deltaId = Encoding.DecodeZigzag(objDataStream)
self.lastObjId += deltaId
objectId = self.lastObjId
#print "objectId", objectId
metaData = self.DecodeMetaData(objDataStream)
refLen = Encoding.DecodeVarint(objDataStream)
#print "len ref", refLen
refStart = objDataStream.tell()
refs = []
while objDataStream.tell() < refStart + refLen:
self.lastRefNode += Encoding.DecodeZigzag(objDataStream)
refs.append(self.lastRefNode)
#print "ref", self.lastRefNode
tags = {}
while objDataStream.tell() < len(objData):
firstString, secondString = self.ReadStringPair(objDataStream)
#print "strlen", len(firstString), len(secondString)
#print "str", firstString.decode("utf-8"), secondString.decode("utf-8")
tags[firstString.decode("utf-8")] = secondString.decode("utf-8")
#print tags
if self.funcStoreWay is not None:
self.funcStoreWay(objectId, metaData, tags, refs)
class CopyPipe(object):
"Splits one big COPY to chunks."
def __init__(self, dstcurs, tablename = None, limit = 512*1024,
sql_from = None):
self.tablename = tablename
self.sql_from = sql_from
self.dstcurs = dstcurs
self.buf = StringIO()
self.limit = limit
#hook for new data, hook func should return new data
#def write_hook(obj, data):
# return data
self.write_hook = None
#hook for flush, hook func result is discarded
# def flush_hook(obj):
# return None
self.flush_hook = None
self.total_rows = 0
self.total_bytes = 0
def write(self, data):
"New data from psycopg"
if self.write_hook:
data = self.write_hook(self, data)
self.total_bytes += len(data)
self.total_rows += data.count("\n")
if self.buf.tell() >= self.limit:
pos = data.find('\n')
if pos >= 0:
# split at newline
p1 = data[:pos + 1]
p2 = data[pos + 1:]
self.buf.write(p1)
self.flush()
data = p2
self.buf.write(data)
def flush(self):
"Send data out."
if self.flush_hook:
self.flush_hook(self)
if self.buf.tell() <= 0:
return
self.buf.seek(0)
if self.sql_from:
self.dstcurs.copy_expert(self.sql_from, self.buf)
else:
self.dstcurs.copy_from(self.buf, self.tablename)
self.buf.seek(0)
self.buf.truncate()
def _flushHeaderMessage(self, buf, wout, wsz):
"""Write a message for self.HEADERS_CLIENT_TYPE
@param buf(StringIO): Buffer to write message to
@param wout(str): Payload
@param wsz(int): Payload length
"""
transform_data = StringIO()
# For now, all transforms don't require data.
num_transforms = len(self.__write_transforms)
for trans_id in self.__write_transforms:
transform_data.write(getVarint(trans_id))
# Add in special flags.
if self.__identity:
self.__write_headers[self.ID_VERSION_HEADER] = self.ID_VERSION
self.__write_headers[self.IDENTITY_HEADER] = self.__identity
info_data = StringIO()
# Write persistent kv-headers
THeaderTransport._flush_info_headers(info_data,
self.get_write_persistent_headers(),
self.INFO_PKEYVALUE)
# Write non-persistent kv-headers
THeaderTransport._flush_info_headers(info_data,
self.__write_headers,
self.INFO_KEYVALUE)
header_data = StringIO()
header_data.write(getVarint(self.__proto_id))
header_data.write(getVarint(num_transforms))
header_size = transform_data.tell() + header_data.tell() + \
info_data.tell()
padding_size = 4 - (header_size % 4)
header_size = header_size + padding_size
wsz += header_size + 10
buf.write(pack(b"!I", wsz))
buf.write(pack(b"!HH", self.HEADER_MAGIC, self.__flags))
buf.write(pack(b"!I", self.__seq_id))
buf.write(pack(b"!H", header_size // 4))
buf.write(header_data.getvalue())
buf.write(transform_data.getvalue())
buf.write(info_data.getvalue())
# Pad out the header with 0x00
for x in range(0, padding_size, 1):
buf.write(pack(b"!c", b'\0'))
# Send data section
buf.write(wout)
class BufferUnPacker(BaseUnPacker):
_head_fmt = '!H'
_head_size = struct.calcsize(_head_fmt)
def __init__(self):
self._buffer = StringIO()
if not hasattr(self, '_head_next_size'):
self._head_next_size = self._head_size - hb_packet_size
def _buffer_size(self):
cur_pos = self._buffer.tell()
self._buffer.seek(0, 2)
size = self._buffer.tell()
self._buffer.seek(cur_pos)
return size - cur_pos
def unpack(self, data):
#往最后添加数据
#log.debug('unpack:%s', repr(data))
global hb_packet_size, hb_pkt
cur_pos = self._buffer.tell()
self._buffer.seek(0, 2)
self._buffer.write(data)
#定位回当前位置
self._buffer.seek(cur_pos)
packages = []
size = self._buffer_size()
while size > self._head_size:
#heartbeat
data = self._buffer.read(hb_packet_size)
if data == hb_pkt:
size -= hb_packet_size
continue
if self._head_next_size:
data = '%s%s' % (data, self._buffer.read(self._head_next_size))
tag, need =struct.unpack(self._head_fmt, data)
#need = struct.unpack(self._head_fmt, self._buffer.read(self._head_size))[0]
if need == 0:#心跳
#log.debug(u'收到心跳包')
size -= self._head_size
continue
size -= (need + self._head_size)
if size < 0:
#定位回头,用于下次读取
self._buffer.seek(-self._head_size, 1)
break
data = self._buffer.read(need)
t = self.raw_unpack(tag, data)
if t is not None:
packages.append(t)
if not size and self._buffer.tell() > 0:
self._buffer.truncate(0)
return packages
def seqChunks(self,chunkSize):
seq = StringIO()
for line in self.seqLines():
seq.write(line.rstrip("\n"))
if seq.tell() >= chunkSize:
yield seq.getvalue()
seq.close()
seq = StringIO()
if seq.tell() > 0:
yield seq.getvalue()
seq.close()
class Buffer(object):
def __init__(self, data=""):
self.buf = StringIO()
if data:
self.buf.write(data)
def __str__(self):
return self.buf.getvalue()
def __len__(self):
o = self.buf.tell()
self.buf.seek(0, 2) # 2 = os.SEEK_END
x = self.buf.tell()
self.buf.seek(o)
return x
def __getitem__(self, i):
# not support slice-step
c = len(self)
if not isinstance(i, slice):
if i >= c or i < -c:
raise IndexError("buffer index out of range")
if i < 0:
i = len(self) + i
i = slice(i, i + 1)
start = i.start
stop = i.stop
if start is None:
start = 0
i = None
if stop is None:
stop = c
i = None
if i is None:
i = slice(start, stop)
o = self.buf.tell()
self.buf.seek(i.start)
data = self.buf.read(i.stop - i.start)
self.buf.seek(o)
return data
def write(self, s):
self.buf.write(s)
def drop(self, n):
# 如果将数据作为StringIO构造函数的参数传递会导致cStringIO构造成StringI，
# 或StringIO构造后pos为0，调用write前需要seek，否则会导致覆盖已有的数据。
buf_new = StringIO()
buf_new.write(self.buf.getvalue()[n:])
self.buf = buf_new
def truncate(self, pos):
self.buf.truncate(pos)
def to_arc(self):
"""Export the Arc object to .arc file format."""
output = BytesIO()
file_count = len(self.__info_table)
# Header
p1_offset = 0x60 + len(self.__data) + 0x4 + len(self.__info_table) * 16
output.write(pack('<4I', 0, p1_offset, file_count, file_count + 3))
output.write(b'\0' * 0x70) # Padding
output.write(self.__data) # Data
output.write(pack('<I', file_count)) # File count
label_offset = p1_offset + file_count * 4 + (file_count + 3) * 8 + 16
name_length = 0
filenames = []
info_offsets = []
# File info table
for file_index in xrange(file_count):
name_offset = label_offset + name_length
info_offsets.append(output.tell() - 0x20)
info = self.__info_table[file_index]
output.write(pack('<4I', name_offset, file_index, info.length,
info.offset))
filenames.append(info.name.encode('shift-jis'))
name_length += len(filenames[file_index]) + 1
# Pointer region 1
for file_index in xrange(file_count):
output.write(pack('<I', info_offsets[file_index]))
# Pointer region 2
output.write(pack('<2I', 0x60, 0x0))
output.write(pack('<2I', 0x60 + len(self.__data), 0x5))
output.write(pack('<2I', 0x60 + len(self.__data) + 0x4, 0xB))
name_length = 16
for file_index in xrange(file_count):
output.write(pack('<2I', info_offsets[file_index], name_length))
name_length += len(filenames[file_index]) + 1
# Label region
output.write(b'Data\0Count\0Info\0')
for file_index in xrange(file_count):
output.write(filenames[file_index] + b'\0')
# File size
size = output.tell()
output.seek(0, os.SEEK_SET)
output.write(pack('<I', size))
return output.getvalue()
def read_framed_message(self):
message = []
while True:
buffer = StringIO()
buffer_length = self._read_buffer_length()
if buffer_length == 0:
return ''.join(message)
while buffer.tell() < buffer_length:
chunk = self.reader.read(buffer_length - buffer.tell())
if chunk == '':
raise ConnectionClosedException("Reader read 0 bytes.")
buffer.write(chunk)
message.append(buffer.getvalue())
class Buffer(object):
def __init__(self):
self.data = StringIO()
self.readonly = False
def __len__(self):
pos = self.data.tell()
self.data.seek(0, os.SEEK_END)
size = self.data.tell()
self.data.seek(pos, os.SEEK_SET)
return size
def getData(self):
return self.data.getvalue()
def setReadOnly(self, readonly):
self.readonly = readonly
def getPos(self):
return self.data.tell()
def setPos(self, pos):
self.data.seek(pos, os.SEEK_SET)
def rewind(self):
self.data.seek(0, os.SEEK_SET)
def skip(self, count):
assert not self.readonly
self.data.seek(count, os.SEEK_CUR)
def write(self, buf):
assert not self.readonly
self.data.write(buf)
def writeByte(self, val):
assert not self.readonly
self.data.write(struct.pack("<B", val))
def writeInt(self, val):
assert not self.readonly
self.data.write(struct.pack("<I", val))
def read(self, count):
return self.data.read(count)
def readByte(self):
return struct.unpack("<B", self.data.read(1))[0]
def readInt(self):
return struct.unpack("<I", self.data.read(4))[0]
def repack_text(filename):
fp = open(filename[:-4].replace("__" , "\\") ,"rb")
data = fp.read()
sig = data[:4]
fp.close()
buffer = StringIO()
buffer.write(data)
buffer.seek(0)
lines = codecs.open("CNtext\\%s"%filename , "rb" , 'utf-16').readlines()
string_list, head_list = makestr_lines(lines)
if sig == "TALD":
buffer.seek(0X8)
index0 = struct.unpack(">I" , buffer.read(4))[0]
index1 = struct.unpack(">I" , buffer.read(4))[0]
buffer.seek(0x20 + index0 * 0x10 + index1 * 0x20 )
buffer.truncate()
buffer.seek(0,2)
for i in xrange(len(string_list)):
string = string_list[i]
string = string.replace("\r" ,"")
string = string.replace("{end}" , u"\u0000")
string = string.encode("utf-16be")
ofs0 = buffer.tell()
buffer.write(string)
buffer.seek(0x20 + index0 * 0x10 + i * 0x20 + 0x14 )
buffer.write(struct.pack(">I" , ofs0))
buffer.seek(0,2)
if sig == "TXTD":
buffer.seek(0X8)
index0 = struct.unpack(">I" , buffer.read(4))[0]
buffer.seek(0x20 + index0 * 0x4)
buffer.truncate()
for i in xrange(len(string_list)):
string = string_list[i]
string = string.replace("\r" ,"")
string = string.replace("{end}" , u"\u0000")
string = string.encode("utf-16be")
ofs0 = buffer.tell()
buffer.write(string)
buffer.seek(0x20 + i * 4)
buffer.write(struct.pack(">I" , ofs0))
buffer.seek(0,2)
data = buffer.getvalue()
folder = os.path.dirname("import\\%s"%filename[:-4].replace("__" , "\\"))
if not os.path.exists(folder):
os.makedirs(folder)
dest = open("import\\%s"%filename[:-4].replace("__" , "\\") ,"wb")
dest.write(data)
if (len(data)%0x80 != 0):
dest.write("\x00" * (0x80 - len(data)%0x80))
dest.close()
def recvall(self, size):
u"""Recebe dados exaustivamente via socket.
Use esta função para receber do socket exatamente `size' bytes.
Levanta uma exceção caso nenhum byte seja recebido.
Nota: em geral, use esta função ou 'recv' ao invés do método 'socket.recv'.
"""
data = StringIO()
while data.tell() < size:
data.write(self.recv(size - data.tell()))
data_str = data.getvalue()
data.close()
return data_str
class Inode(py9p.Dir):
"""
VFS inode, based on py9p.Dir
"""
def __init__(self,name,parent,qtype=0):
py9p.Dir.__init__(self,True)
self.parent = parent
self.name = name
#
# DMDIR = 0x80000000
# QTDIR = 0x80
#
self.qid = py9p.Qid((qtype >> 24) & py9p.QTDIR, 0, py9p.hash8(self.absolute_name()))
self.type = 0
self.dev = 0
self.atime = self.mtime = int(time.time())
self.uidnum = self.muidnum = os.getuid()
self.gidnum = os.getgid()
self.uid = self.muid = pwd.getpwuid(self.uidnum).pw_name
self.gid = grp.getgrgid(self.gidnum).gr_name
self.children = {}
self.writelock = False
if self.qid.type & py9p.QTDIR:
self.mode = py9p.DMDIR | DEFAULT_DIR_MODE
self.children["."] = self
self.children[".."] = self.parent
else:
self.mode = DEFAULT_FILE_MODE
self.data = StringIO()
def absolute_name(self):
if (self.parent is not None) and (self.parent != self):
return "%s/%s" % (self.parent.absolute_name(),self.name)
else:
return self.name
def sync(self):
pass
@property
def length(self):
if self.qid.type & py9p.QTDIR:
return len(self.children.keys())
else:
p = self.data.tell()
self.data.seek(0,os.SEEK_END)
l = self.data.tell()
self.data.seek(p,os.SEEK_SET)
return l
class BufferedCompressedReader(object):
def __init__(self, inputstream, buffer_size=_DEFAULT_BUFFER_SIZE):
self.inputstream = inputstream
self.decompressor = zlib.decompressobj()
self.compressed_buffer = StringIO()
self.decompressed_buffer = StringIO()
self.buffer_size = buffer_size
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.close()
def close(self):
self.decompressor = None
self.compressed_buffer = None
self.decompressed_buffer = None
def read(self, byte_count=None):
eof = False
while not eof and self.decompressed_buffer.tell() < byte_count:
# make sure the number of compressed bytes is at least as large as then number of decompressed bytes we still need
while not eof and self.compressed_buffer.tell() < byte_count - self.decompressed_buffer.tell():
new_compressed_bytes = self.inputstream.read(self.buffer_size)
if not new_compressed_bytes:
eof = True
else:
self.compressed_buffer.write(new_compressed_bytes)
compressed_bytes = self.compressed_buffer.getvalue()
self.compressed_buffer.truncate(0)
new_uncompressed_bytes = self.decompressor.decompress(compressed_bytes, byte_count - self.decompressed_buffer.tell())
self.decompressed_buffer.write(new_uncompressed_bytes)
if self.decompressor.unconsumed_tail:
self.compressed_buffer.write(self.decompressor.unconsumed_tail)
elif self.decompressor.unused_data:
self.compressed_buffer.write(self.decompressor.unused_data)
# file may be a sequence of separate compression streams.
# if we hit unused_data then we're at a new stream and
# need to reset the decompressor.
self.decompressor = zlib.decompressobj()
decompressed = self.decompressed_buffer.getvalue()
self.decompressed_buffer.close()
self.decompressed_buffer = StringIO()
self.decompressed_buffer.write(decompressed[byte_count:])
return decompressed[:byte_count]
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, 'No permission to list directory')
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write('<html>\n<title>Directory listing for %s</title>\n' %
displaypath)
f.write('<body>\n<h2>Directory listing for %s</h2>\n' % displaypath)
f.write('<hr>\n<ul>\n')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
if os.path.isdir(fullname):
displayname = name + '/'
linkname = name + '/'
if os.path.islink(fullname):
displayname = name + '@'
f.write('<li><a href="%s">%s</a>\n' %
(urllib.quote(linkname), cgi.escape(displayname)))
f.write('</ul>\n<hr>\n</body>\n</html>\n')
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header('Content-type', 'text/html; charset=%s' % encoding)
self.send_header('Content-Length', str(length))
self.end_headers()
return f
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
"""f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n<ul>\n")"""
f.write('(')
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write("'%s'," % cgi.escape(displayname))
f.write(')')
#f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
class ConditionalFile:
def __init__(self, filename, hashfunc):
self.filename = filename
self.hashfunc = hashfunc
self._buf = StringIO()
def write(self, data):
self._buf.write(data)
def read(self):
return self._buf.read()
def seek(self, *args):
self._buf.seek(*args)
def tell(self):
return self._buf.tell()
def close(self):
self._buf.seek(0)
newhash = self.hashfunc(self._buf)
if not os.path.isfile(self.filename):
self.__write()
return (self.filename, newhash, False)
f = open(self.filename, 'rb')
oldhash = self.hashfunc(f)
self.seek(0)
f.close()
modified = oldhash != newhash
if modified:
self.__write()
return (self.filename, newhash, modified)
def __write(self):
self.seek(0)
f = open(self.filename, 'wb')
f.write(self._buf.read())
f.close()
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
list = os.listdir(path)
except os.error:
self.send_error(404, "No permission to list directory")
return None
list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" % displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n")
f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
f.write("<input name=\"file\" type=\"file\"/>")
f.write("<input type=\"submit\" value=\"upload\"/></form>\n")
f.write("<hr>\n<ul>\n")
for name in list:
fullname = os.path.join(path, name)
displayname = linkname = name
# Append / for directories or @ for symbolic links
if os.path.isdir(fullname):
displayname = name + "/"
linkname = name + "/"
if os.path.islink(fullname):
displayname = name + "@"
# Note: a link to a directory displays with @ and links with /
f.write('<li><a href="%s">%s</a>\n'
% (urllib.quote(linkname), cgi.escape(displayname)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def create_index_record(self, secondary=False): # {{{
header_length = 192
buf = StringIO()
indices = list(SecondaryIndexEntry.entries()) if secondary else self.indices
# Write index entries
offsets = []
for i in indices:
offsets.append(buf.tell())
buf.write(i.bytestring)
index_block = align_block(buf.getvalue())
# Write offsets to index entries as an IDXT block
idxt_block = b'IDXT'
buf.seek(0), buf.truncate(0)
for offset in offsets:
buf.write(pack(b'>H', header_length+offset))
idxt_block = align_block(idxt_block + buf.getvalue())
body = index_block + idxt_block
header = b'INDX'
buf.seek(0), buf.truncate(0)
buf.write(pack(b'>I', header_length))
buf.write(b'\0'*4) # Unknown
buf.write(pack(b'>I', 1)) # Header type? Or index record number?
buf.write(b'\0'*4) # Unknown
# IDXT block offset
buf.write(pack(b'>I', header_length + len(index_block)))
# Number of index entries
buf.write(pack(b'>I', len(offsets)))
# Unknown
buf.write(b'\xff'*8)
# Unknown
buf.write(b'\0'*156)
header += buf.getvalue()
ans = header + body
if len(ans) > 0x10000:
raise ValueError('Too many entries (%d) in the TOC'%len(offsets))
return ans
class LatexBuffer(object):
""" Verifying LaTeX buffer """
__rePhrases = [re.compile(p) for p in [
'\b',
r'\sexttt\{.+\}',
]]
def __init__(self):
self._buf = StringIO()
self._verbatim = False
def tell(self):
return self._buf.tell()
def write(self, txt, validate=True):
"""
Write text to buffer after, after optionally checking
whether LaTeX backslashes are there or raw strings have
been forgotten.
"""
if validate:
for rp in self.__rePhrases:
#print 'checking "%s" in %s' % (p, string)
if rp.search(txt):
raise ValueError, \
'This text,\n----\n%s\n----\ncontains '\
'LaTeX commands but was not typed as a raw string' \
% txt
# Why this??? Nothing happens and actions for verbatim envirs
# are taken in _verbatim_text
if not self._verbatim:
phrases = _verbatim_phrases()
for p in phrases:
if txt.find(p) != -1:
self._verbatim = True
break
self._buf.write(txt)
def get_text(self):
return self._buf.getvalue()
def noindex(self, path):
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write(
'\n<!-- This File was automaticly generated by Locks Web Server-->\n'
)
f.write('\n<!-- Produced by Core 4 The Lan-->\n')
f.write('<html>\n<title>Index for {}</title>\n'.format(displaypath))
f.write(
'<body>\n<h2>Non-Existant Index for {}</h2>\n<h3>No index here...</h3>\n'
.format(displaypath))
f.write(
'<p>Soz, the person in charge of maintaining the Web Site\'s Content Side</p>\n'
)
f.write(
'<p>of the server, cannot keep up with his workload (as you can see) and</p>\n'
)
f.write(
'<p>has not made a index page for this directory. <b>Instead</b> <i>I</i> </p>\n'
)
f.write(
'<p>(The writer of this software - Locks - The best Web Server Software out</p>\n'
)
f.write(
'<p>there at the moment*) has to program this server to generate this page</p>\n'
)
f.write(
'<p>and <u>therefore</u> <b>LAGGING</b> this Extremly <i>Fast</i> Server. </p>\n'
)
f.write(
'</br>\n<p>*and best of all - It\'s writen in <a href=http://www.python.org>Python</a></p>'
)
f.write('\n<hr>\n<p>{}</p></body></html>'.format(self.poweredby))
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def write_lkv_socket(sock, dct):
""" Convert a dictionary of strings to an LKV message and write it to the
socket. """
sio = StringIO()
sio.write('\x00'*4) # reserve space for length header
sio.write(np.array(len(dct), dtype='=u4').tostring()) # array length header
for name, value in dct.iteritems():
write_packed_string(sio, name)
write_packed_string(sio, value)
# Write length header
msg_len = sio.tell() - 4
sio.seek(0)
sio.write(np.array(msg_len, dtype='=u4').tostring())
sio.seek(0)
sock.sendall(sio.read())
def serve_home(self, path):
"""
Produce the home page with a form to allow an upload.
"""
f = StringIO()
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Upload a file</title>\n")
f.write("<body>\n<h2>Upload a file</h2>\n")
f.write("<hr>\n")
f.write("<form ENCTYPE=\"multipart/form-data\" method=\"post\">")
f.write("<input name=\"file\" type=\"file\"/>")
f.write("<input type=\"submit\" value=\"upload\"/></form>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def write_packet(b, pkt, compression_threshold):
raw = StringIO()
write_varint(raw, pkt.id)
pkt.emit(raw)
size = raw.tell()
if compression_threshold is None:
write_varint(b, size)
b.write(raw.getvalue())
elif size >= compression_threshold:
data = raw.getvalue()
compressed = data.encode('zlib')
write_varint(b, size_varint(len(data)) + len(compressed))
write_varint(b, len(data))
b.write(compressed)
else:
data = raw.getvalue()
data_size = size_varint(0) + len(data)
write_varint(b, data_size)
write_varint(b, 0)
b.write(data)
def convertWithPdfMiner(fname):
pages_text = []
rsrcmgr = PDFResourceManager()
sio = StringIO()
codec = 'utf-8' # ISO-8859-1 is good for foreign languages
laparams = LAParams()
device = TextConverter(rsrcmgr, sio, codec=codec, laparams=laparams)
interpreter = PDFPageInterpreter(rsrcmgr, device)
pdf = open(fname, "rb")
count = 0
for page in PDFPage.get_pages(pdf):
# Get (and store) the "cursor" position of stream before reading from PDF
# On the first page, this will be zero
read_position = sio.tell()
interpreter.process_page(page)
sio.seek(read_position, 0)
page_text = sio.read()
pages_text.append(page_text)
return pages_text
def wrapped_secrets(self, wrapping_key):
'''Marshal the secrets into their stored format, optionally wrapping them with the supplied key.'''
buf = BytesIO()
for secret in self.secrets:
slotdata = secret.contents
assert (len(slotdata) % 4) == 0
buf.write(struct.pack('>BBH', secret.tp, len(slotdata)//4, secret.id))
buf.write(slotdata)
# Apply padding as necessary for AESWRAP
fragment = buf.tell() % 8
if fragment > 0:
buf.write( b'\x00' * (8 - fragment) )
marshalled = buf.getvalue()
if wrapping_key is None:
return marshalled
else:
return primitives.keywrap.aes_key_wrap(wrapping_key,
marshalled,
backend)
def check_auth(self):
if not options.get('auth'):
return True
au = self.headers.getheader('authorization')
if au and len(au) > 6 and au.endswith(options.get('auth')):
return True
f = StringIO()
f.write('<center><h2>401 Unauthorized</h2></center>')
self.send_response(401, "Unauthorized")
self.send_header("Content-Type", "text/html")
self.send_header("Content-Length", str(f.tell()))
self.send_header(
"WWW-Authenticate",
'Basic realm="%s"' % (options.get('realm') or self.server_version))
self.send_header('Connection', 'close')
self.end_headers()
f.seek(0)
shutil.copyfileobj(f, self.wfile)
return False
def do_POST(self):
"""Serve a POST request."""
r, info = self.deal_post_data()
print r, info, "by: ", self.client_address
f = StringIO()
if r:
f.write("<strong>Success:</strong>")
else:
f.write("<strong>Failed:</strong>")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def __disassemble(code, mask, opcode_dict, start_address):
stream = StringIO(code)
uncharted_opcode_translation_table = open(
'vm1_uncharted_opcode_translation.txt',
'rU').read().strip().decode('hex')
disallow_masking = [
line.strip()
for line in open('vm2_require_nativity_mask.txt', 'rU').readlines()
]
while stream.tell() < len(code):
opcode_lo = stream.read(1)
stream.seek(-1, 1)
if is_opcode_in_table(opcode_lo, opcode_dict):
if chr(ord(opcode_lo)
^ 0x8a).encode('hex') not in disallow_masking:
if mask[stream.tell()] == '\x00':
yield disassemble_native(
stream, uncharted_opcode_translation_table, len(code),
start_address + stream.tell())
else:
try:
yield disassemble_bytecode(
stream, opcode_dict, start_address + stream.tell())
except IllegalBytecode:
yield disassemble_native(
stream, uncharted_opcode_translation_table,
len(code), start_address + stream.tell())
else:
try:
yield disassemble_bytecode(stream, opcode_dict,
start_address + stream.tell())
except IllegalBytecode:
yield disassemble_native(
stream, uncharted_opcode_translation_table, len(code),
start_address + stream.tell())
else:
yield disassemble_native(stream,
uncharted_opcode_translation_table,
len(code), start_address + stream.tell())
def clean_photo(self):
"""Clean possible bad Image data.
Try to load EXIF data from image. If that fails, remove EXIF
data by re-saving the image. Related bug 919736.
"""
photo = self.cleaned_data['photo']
if photo and isinstance(photo, UploadedFile):
image = Image.open(photo.file)
try:
image._get_exif()
except (AttributeError, IOError, KeyError, IndexError):
cleaned_photo = StringIO()
if image.mode != 'RGB':
image = image.convert('RGB')
image.save(cleaned_photo, format='JPEG', quality=95)
photo.file = cleaned_photo
photo.size = cleaned_photo.tell()
return photo
class SmartIO:
"""
The SmartIO class allows one to put a cap on the memory consumption.
StringIO objects are very fast, because they are stored in memory, but
if they are too big the memory footprint becomes noticeable.
The write method of a SmartIO determines if the data that is to be added
to the (initially) StrintIO object does not exceed a certain threshold; if
it does, it switches the storage to a temporary disk file
"""
def __init__(self, max_mem_size=16384, force_mem=0):
self._max_mem_size = max_mem_size
self._io = StringIO()
# self._fixed is a flag to show if we're supposed to consider moving
# the StringIO object into a tempfile
# Invariant: if self._fixed == 0, we have a StringIO (if self._fixed
# is 1 and force_mem was 0, then we have a file)
if force_mem:
self._fixed = 1
else:
self._fixed = 0
def set_max_mem_size(self, max_mem_size):
self._max_mem_size = max_mem_size
def get_max_mem_size(self):
return self._max_mem_size
def write(self, data):
if not self._fixed:
# let's consider moving it to a file
if len(data) + self._io.tell() > self._max_mem_size:
# We'll overflow, change to a tempfile
tmpfile = _tempfile()
tmpfile.write(self._io.getvalue())
self._fixed = 1
self._io = tmpfile
self._io.write(data)
def __getattr__(self, name):
return getattr(self._io, name)
def recv_msg(self, host, callback):
while True:
try:
self.alive_mutex.acquire()
try:
sock = self.alive[host]
except KeyError:
logging.debug('{} died while in recv!'.format(host))
self.alive_mutex.release()
break
self.alive_mutex.release()
bytelen = sock.recv(4)
if len(bytelen) == 0:
self.handle_crash(host)
break
picklen = unpack('>I', bytelen)[0]
pickled = sock.recv(picklen)
numbytes = len(pickled)
logging.debug('Recv got {} bytes'.format(numbytes))
if numbytes == 0:
self.handle_crash(host)
break
pbuf = StringIO(pickled)
while pbuf.tell() < numbytes:
message = cPickle.load(pbuf)
self.handle_message(message)
time.sleep(0.5)
except socket.timeout:
time.sleep(1)
continue
except socket.error as e:
logging.warning('Socket error: {}'.format(e.errno))
self.handle_crash(host)
break
def do_POST(self):
"""Serve a POST request."""
decision = 'Unknown'
r, info, img_file, prediction = self.deal_post_data()
print("prediction", prediction)
if (prediction > 0.5):
decision = 'REJECTED'
elif (prediction <= 0.5):
decision = "PASSED"
print r, info, "by: ", self.client_address
f = StringIO()
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Upload Result Page</title>\n")
f.write("<body>\n<h2>Upload Result Page</h2>\n")
f.write('<img src=' + img_file + ' height=100 width=100 >\n')
f.write("<hr>\n")
if r:
f.write("<strong>Success:</strong>")
else:
f.write("<strong>Failed:</strong>")
f.write("<h3><strong> The decision is <red>" + decision +
"</red></strong></h3>")
f.write(info)
request_params = "correct?" + img_file[1:] + "=" + str(
1 if prediction > 0.5 else 0)
print request_params
f.write("<br><a href='" + self.headers['referer'] + request_params +
"'>Decision is wrong.</a>")
f.write("<br><a href=\"%s\">back</a>" % self.headers['referer'])
f.write("<br><small>Powered By: [email protected].")
f.write("</small></body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_POST(self):
"""Serve a POST request."""
r, info = self.deal_post_data()
print r, info, "by: ", self.client_address
f = StringIO()
image_types = ['.png', '.jpg', '.gif', '.jpeg']
#f.write('File successfully pushed.')
#f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
#f.write("<html>\n<title>Upload Result Page</title>\n")
#f.write("<body>\n<h2>Upload Result Page</h2>\n")
#f.write("<hr>\n")
if r:
if glob.fn.endswith('.ppm'):
leds.uploadPPM()
f.write("Success: PPM file successfully pushed.\n")
elif glob.fn.endswith('.txt'):
leds.uploadTXT()
f.write("Success: TXT file successfully pushed.\n")
elif any(
glob.fn.endswith(image_type)
for image_type in image_types):
leds.uploadImage()
f.write("Success: Image file successfuly pushed.\n")
else:
f.write("Failure: Invalid file type.")
#_, ext = os.path.splitext(glob.fn)
#print "File type: " + ext
#f.write("<strong>Success:</strong>")
else:
f.write("Failure: Failed to push file.\n")
#f.write("<strong>Failed:</strong>")
f.write(info)
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def do_POST(self):
form = cgi.FieldStorage(fp=self.rfile,
headers=self.headers,
environ={
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE':
self.headers['Content-Type'],
})
path = form['path'].value
try:
list = os.listdir(path)
except os.error:
list = []
objs = []
for name in list:
obj = dict()
objs.append(obj)
fullname = (path if path != '/' else '') + '/' + name
obj['id'] = fullname
obj['text'] = name
if os.path.isdir(fullname):
obj['children'] = True
else:
obj['icon'] = "jstree-file"
f = StringIO()
f.write(json.dumps(objs))
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "json charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
shutil.copyfileobj(f, self.wfile)
f.close()
def display_source(self, path):
out = StringIO()
out.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
out.write("<html>\n<title>%s</title>\n" % os.path.basename(path))
out.write(
"<script src=\"https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js\"></script>"
)
out.write('<body>')
if 'referer' in self.headers:
out.write("<a href=\"%s\">Go Back</a>" % self.headers['referer'])
out.write('<pre class=\"prettyprint linenums\">\n')
try:
# Always read in binary mode. Opening files in text mode may cause
# newline translations, making the actual size of the content
# transmitted *less* than the content-length!
f = open(path, 'rb')
fs = os.fstat(f.fileno())
content = f.read()
out.write(cgi.escape(content))
except IOError:
self.send_error(404, "File not found")
return None
try:
out.write("</pre>\n</body>\n</html>\n")
linkname = '/'.join(['raw'] + path.split('/')[1:])
out.write("<br/><a href=\"%s\">Download file</a>" %
urllib.quote(linkname))
length = out.tell()
out.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type",
"text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.send_header("Last-Modified",
self.date_time_string(fs.st_mtime))
self.end_headers()
return out
except:
f.close()
raise
def list_directory(self, path):
"""Helper to produce a directory listing (absent index.html).
Return value is either a file object, or None (indicating an
error). In either case, the headers are sent, making the
interface the same as for send_head().
"""
try:
dir_paths = self._fs.listdir(path, dirs_only=True)
file_paths = self._fs.listdir(path, files_only=True)
except FSError:
self.send_error(404, "No permission to list directory")
return None
paths = [p + '/' for p in sorted(dir_paths, key=lambda p: p.lower())
] + sorted(file_paths, key=lambda p: p.lower())
#list.sort(key=lambda a: a.lower())
f = StringIO()
displaypath = cgi.escape(urllib.unquote(self.path))
f.write('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">')
f.write("<html>\n<title>Directory listing for %s</title>\n" %
displaypath)
f.write("<body>\n<h2>Directory listing for %s</h2>\n" % displaypath)
f.write("<hr>\n<ul>\n")
parent = dirname(path)
if path != parent:
f.write('<li><a href="%s">../</a></li>' %
urllib.quote(parent.rstrip('/') + '/'))
for path in paths:
f.write('<li><a href="%s">%s</a>\n' %
(urllib.quote(path), cgi.escape(path)))
f.write("</ul>\n<hr>\n</body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html")
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def handler(request):
html = StringIO()
html.write("{")
first = True
for header in request.headers:
val = request.headers[header]
if not first:
html.write(",")
first = False
html.write("\"%s\": \"%s\"" % (header, val))
html.write("}")
request.send_response(200)
request.send_header("Content-type", "application/json")
request.send_header("Content-Length", html.tell())
request.send_header("Cache-Control", "no-cache")
request.end_headers()
request.wfile.write(html.getvalue())
def _sendresponse(self, code, message):
"""Send a response of type 'text/plain'
Args:
code (int): Return status code
message (str): Body of the return message
"""
from cStringIO import StringIO
f = StringIO()
f.write(message)
length = f.tell()
f.seek(0)
self.send_response(code)
self.send_header("Content-type", "text/plain; charset=utf8")
self.send_header("Content-Length", str(length))
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
import shutil
shutil.copyfileobj(f, self.wfile)
def parse_list(cls, s):
n = len(s)
f = StringIO(s)
last_type = None
while f.tell() < n:
dt = midi_unpack_varlen(f)
type, = midi_unpack('B', f)
if type == 0xFF:
me = MidiMetaEvent.parse(f)
elif type >= 0xF0:
me = MidiSystemEvent.parse(f, type)
elif type >= 0x80:
me = MidiChannelEvent.parse(f, type)
last_type = type
else:
# running status
assert last_type != None, "running status without previous status"
midi_pack_back('B', f)
me = MidiChannelEvent.parse(f, last_type)
me.dt = dt
yield me
def write_response(response, outgoing_buf):
content_buf = StringIO()
#gzip stream
header = response["header"]
if "Vary" in header:
header["Vary"] += ", Accept-Encoding"
else:
header["Vary"] = "Accept-Encoding"
if "gzip" in header.get("Content-Encoding", ""):
gzip_write(response["stream"], content_buf)
else:
content_buf.write(response["stream"])
header["Content-Length"] = str(content_buf.tell())
#add cookie to header
if "cookie" in response:
cookie = simple_http.unparse_full_cookie(response["cookie"])
header["Set-Cookie"] = cookie
data = simple_http.unparse_header(header, False)
#write header and content
outgoing_buf.write("".join((data, NEWLINE, content_buf.getvalue())))
content_buf.close()
def send_datalist(self):
l = []
for path, subs, files in os.walk(visualizer_path):
for file in files:
mtime = os.stat('%s/%s' % (path, file))[8]
l.append(
'{"path":["%s"],"name":"%s","mtime":%d}' % ('","'.join(
path[len(visualizer_path):].split('/')), file, mtime))
l.sort()
f = StringIO()
f.write("[")
f.write(','.join(l))
f.write("]")
length = f.tell()
f.seek(0)
self.send_response(200)
encoding = sys.getfilesystemencoding()
self.send_header("Content-type", "text/html; charset=%s" % encoding)
self.send_header("Content-Length", str(length))
self.end_headers()
return f
def test_pickle_roundtrip():
kv_pairs = [(b'key 1', b'value 1'), (b'hello', b'world'), (b'the', b'end')]
# Each key or value requires 4 bytes for a length field.
# Pickling a bytes object requires 7 bytes plus the length.
expected_size = 4 + sum(11 + len(k) + 11 + len(v) for k, v in kv_pairs)
f = BytesIO()
writer = BinWriter(f)
for pair in kv_pairs:
writer.writepair(pair)
writer.finish()
size = f.tell()
assert size == expected_size
f.seek(0)
reader = BinReader(f)
new_pairs = list(reader)
assert new_pairs == kv_pairs
def test_raw_roundtrip():
kv_pairs = [(b'key 1', b'value 1'), (b'hello', b'world'), (b'the', b'end')]
expected_size = 4 + sum(4 + len(k) + 4 + len(v) for k, v in kv_pairs)
serializers = Serializers(raw_serializer, '', raw_serializer, '')
f = BytesIO()
writer = BinWriter(f, serializers=serializers)
for pair in kv_pairs:
writer.writepair(pair)
writer.finish()
size = f.tell()
assert size == expected_size
f.seek(0)
reader = BinReader(f, serializers=serializers)
new_pairs = list(reader)
assert new_pairs == kv_pairs