def test_is_xml(self):
xmlfile = StringIO('<?xml version="1.0"?>\n<tag>\n')
htmlfile = StringIO('<html>\n<head></head>\n<body><p></p></body>\n</html>')
self.assertEqual(summary.isxml_stream(xmlfile), True)
self.assertEqual(xmlfile.tell(), 0)
self.assertEqual(summary.isxml_stream(htmlfile), False)
class TestStreamStructure(TestInMemoryStructure):
def json(self, head):
return jl(
jd(
(q(self.LIST_TOKEN), jl(*(q(w) for w in self.LIST))),
(q(self.DICT_TOKEN), jd(*((q(k), q(v)) for k,v in self.DICT.items()))),
(q(self.HEAD_TOKEN), jd(*((q(k), q(v)) for k,v in self.HEAD.items()))),
),
q(head),
)
def get_read_store(self):
self.stream = StringIO(self.json(self.HEAD_TOKEN))
return structure.JSONStreamReadStructure(self.stream)
def get_write_store(self):
self.stream = StringIO()
return structure.JSONStreamWriteStructure(self.stream)
def verify_dict(self, received, expect):
tools.assert_equal(dict(expect), received)
def verify_list(self, received, expect):
tools.assert_equal(list(expect), received)
def verify_write(self, store, head):
# Nothing written to stream until close().
tools.assert_equal(0, self.stream.tell())
tools.assert_equal('', self.stream.read())
store.close()
self.stream.seek(0)
tools.assert_equal(self.json(head), self.stream.read())
def test_no_close(self):
store = self.get_read_store()
tools.assert_raises(AttributeError, getattr, store, 'close')
def open(self,f=None,mode='r',encrypt=False,sign=False):
"""write: encrypt = list of recipients, sign = sender or bool(default_key)
read: encrypt = encrypted data expected, sign= expected key or True=defaultkey
"""
from six.moves import StringIO
from six import next, PY3, BytesIO
self.mode,self.encrypt,self.sign = mode,encrypt,sign
if self.required==True: self.required = self.columns
if encrypt or sign: assert self.gpg, 'gpg not intialized'
self.origfile = self.file = f
assert mode in ('r','w'), 'invalid mode'
if mode=='r':
if sign:
if sign==True:
fingerprint = self.gpg.default_key
if type(fingerprint) == tuple: fingerprint = fingerprint[0]
else:
if type(sign) == tuple: sign = sign[0]
fingerprint = self.gpg.find_key(sign)
assert fingerprint, "sender key not found"
if self.fileformat=='csv':
import re
if encrypt:
if PY3 and isinstance(f,StringIO):
result = self.gpg.decrypt_str(f.getvalue())
else:
result = self.gpg.decrypt_file(f)
assert result.ok, "decryption failed"
if sign: assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
f = StringIO(str(result))
elif sign:
if PY3 and isinstance(f,StringIO):
result = self.gpg.verify_str(f.getvalue())
f = StringIO(self.gpg.without_signature(f.getvalue()))
else:
result = self.gpg.verify_file(f)
f.seek(0)
f = StringIO(self.gpg.without_signature(f.read()))
assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
self.file = f
dialect = self.dialect
if not dialect:
pos = f.tell()
dialect = csv.Sniffer().sniff(f.read(1024))
f.seek(pos) # rewind
reader = csv.reader(f,dialect=dialect)
preamble = next(reader)
assert len(preamble), 'invalid file format'
assert preamble[0]==self.dataformat, "file format not supported"
preamble = re.match(r'^(\d+).(\d+)',preamble[1])
assert int(preamble.group(2))<=self.version[0], "format version not supported"
fields = next(reader)
self.csv = reader
else: # self.fileformat in ('json','jsondict','json-file','jsondict-file'):
import json
if self.fileformat in ('json-file','jsondict-file'):
self.file = f = json.load(f)
data, encrypted, signed, result = json_decrypt(f,self.gpg)
assert data, 'invalid input'
if encrypt: assert encrypted==bool(encrypt), 'encryption expected'
if sign:
assert signed==bool(sign), 'signature expected'
assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
assert 'format' in data and data['format']==self.dataformat, "file format not supported"
assert 'version' in data and data['version'][0]<=self.version[0], "file version not supported"
assert 'fields' in data , "fields missing"
fields = data['fields']
self.rows = data['data']
columns, unknown = [], []
for field in fields:
if field in self.columns: columns.append(field)
elif self.ignore: unknown.append(field)
else: assert False, "unknown field '%s'" % field
if self.required:
for field in self.required:
assert field in columns, "missing required field '%s'" % field
self.fields = fields
self.read_columns = (columns,unknown)
elif mode=='w':
assert self.fileformat in ('json','jsondict') or self.file, 'file missing'
if self.fileformat=='csv':
if encrypt or sign: self.file = StringIO()
else: self.file = f
self.csv = csv.writer(self.file,lineterminator='\n',dialect=self.dialect)
self.csv.writerow((self.dataformat,'%i.%i' % tuple(self.version)))
self.csv.writerow(self.columns)
else: # self.fileformat in ('json','jsondict'):
self.rows = []
class InputFile(object):
max_buffer_size = 1024 * 1024
def __init__(self, rfile, length):
"""File-like object used to provide a seekable view of request body data"""
self._file = rfile
self.length = length
self._file_position = 0
if length > self.max_buffer_size:
self._buf = tempfile.TemporaryFile()
else:
self._buf = StringIO()
@property
def _buf_position(self):
rv = self._buf.tell()
assert rv <= self._file_position
return rv
def read(self, bytes=-1):
assert self._buf_position <= self._file_position
if bytes < 0:
bytes = self.length - self._buf_position
bytes_remaining = min(bytes, self.length - self._buf_position)
if bytes_remaining == 0:
return ""
if self._buf_position != self._file_position:
buf_bytes = min(bytes_remaining,
self._file_position - self._buf_position)
old_data = self._buf.read(buf_bytes)
bytes_remaining -= buf_bytes
else:
old_data = ""
assert bytes_remaining == 0 or self._buf_position == self._file_position, (
"Before reading buffer position (%i) didn't match file position (%i)"
% (self._buf_position, self._file_position))
new_data = self._file.read(bytes_remaining)
self._buf.write(new_data)
self._file_position += bytes_remaining
assert bytes_remaining == 0 or self._buf_position == self._file_position, (
"After reading buffer position (%i) didn't match file position (%i)"
% (self._buf_position, self._file_position))
return old_data + new_data
def tell(self):
return self._buf_position
def seek(self, offset):
if offset > self.length or offset < 0:
raise ValueError
if offset <= self._file_position:
self._buf.seek(offset)
else:
self.read(offset - self._file_position)
def readline(self, max_bytes=None):
if max_bytes is None:
max_bytes = self.length - self._buf_position
if self._buf_position < self._file_position:
data = self._buf.readline(max_bytes)
if data.endswith("\n") or len(data) == max_bytes:
return data
else:
data = ""
assert self._buf_position == self._file_position
initial_position = self._file_position
found = False
buf = []
max_bytes -= len(data)
while not found:
readahead = self.read(min(2, max_bytes))
max_bytes -= len(readahead)
for i, c in enumerate(readahead):
if c == "\n":
buf.append(readahead[:i + 1])
found = True
break
if not found:
buf.append(readahead)
if not readahead or not max_bytes:
break
new_data = "".join(buf)
data += new_data
self.seek(initial_position + len(new_data))
return data
def readlines(self):
rv = []
while True:
data = self.readline()
if data:
rv.append(data)
else:
break
return rv
def next(self):
data = self.readline()
if data:
return data
else:
raise StopIteration
def __iter__(self):
return self
def open(self,f=None,mode='r',encrypt=False,sign=False):
"""write: encrypt =bool or list of recipients, sign = sender or bool(default_key)
read: encrypt = bool encrypted data expected, sign= expected key or True=defaultkey
"""
from six.moves import StringIO
from six import next, PY3, BytesIO
self.mode,self.encrypt,self.sign = mode,encrypt,sign
if self.required is True: self.required = self.columns
if encrypt or sign: assert self.gpg, 'gpg not intialized'
self.origfile = self.file = f
assert mode in ('r','w'), 'invalid mode'
if mode=='r':
if sign:
if sign is True:
fingerprint = self.gpg.default_key
if type(fingerprint) == tuple: fingerprint = fingerprint[0]
else:
if type(sign) == tuple: sign = sign[0]
fingerprint = self.gpg.find_key(sign)
assert fingerprint, "sender key not found"
if self.fileformat=='csv':
import re
if encrypt:
if PY3 and isinstance(f,StringIO):
result = self.gpg.decrypt_str(f.getvalue())
else:
result = self.gpg.decrypt_file(f)
assert result.ok, "decryption failed"
if sign: assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
f = StringIO(str(result))
elif sign:
if PY3 and isinstance(f,StringIO):
result = self.gpg.verify_str(f.getvalue())
f = StringIO(self.gpg.without_signature(f.getvalue()))
else:
result = self.gpg.verify_file(f)
f.seek(0)
f = StringIO(self.gpg.without_signature(f.read()))
assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
self.file = f
dialect = self.dialect
if not dialect:
pos = f.tell()
dialect = csv.Sniffer().sniff(f.read(1024))
f.seek(pos) # rewind
if not PY3:
import unicodecsv
reader = unicodecsv.reader
else:
reader = csv.reader
reader = reader(f,dialect=dialect)
preamble = next(reader)
assert len(preamble), 'invalid file format'
assert preamble[0]==self.dataformat, "file format not supported"
preamble = re.match(r'^(\d+).(\d+)',preamble[1])
assert int(preamble.group(2))<=self.version[0], "format version not supported"
fields = next(reader)
self.csv = reader
else: # self.fileformat in ('json','jsondict','json-file','jsondict-file'):
import json
if self.fileformat in ('json-file','jsondict-file'):
self.file = f = json.load(f)
data, encrypted, signed, result = json_decrypt(f,self.gpg)
assert data, 'invalid input'
if encrypt: assert encrypted==bool(encrypt), 'encryption expected'
if sign:
assert signed==bool(sign), 'signature expected'
assert result.valid and result.fingerprint==fingerprint, 'invalid signature'
assert 'format' in data and data['format']==self.dataformat, "file format not supported"
assert 'version' in data and data['version'][0]<=self.version[0], "file version not supported"
assert 'fields' in data , "fields missing"
fields = data['fields']
self.rows = data['data']
columns, unknown = [], []
for field in fields:
if field in self.columns: columns.append(field)
elif self.ignore: unknown.append(field)
else: assert False, "unknown field '%s'" % field
if self.required:
for field in self.required:
assert field in columns, "missing required field '%s'" % field
self.fields = fields
self.read_columns = (columns,unknown)
elif mode=='w':
assert self.fileformat in ('json','jsondict') or self.file, 'file missing'
if self.fileformat=='csv':
if encrypt or sign: self.file = StringIO()
else: self.file = f
if not PY3:
import unicodecsv
writer = unicodecsv.writer
else:
writer = csv.writer
self.csv = writer(self.file,lineterminator='\n',dialect=self.dialect)
self.csv.writerow((self.dataformat,'%i.%i' % tuple(self.version)))
self.csv.writerow(self.columns)
else: # self.fileformat in ('json','jsondict'):
self.rows = []
