def test_is_xml(self): xmlfile = StringIO('<?xml version="1.0"?>\n<tag>\n') htmlfile = StringIO('<html>\n<head></head>\n<body><p></p></body>\n</html>') self.assertEqual(summary.isxml_stream(xmlfile), True) self.assertEqual(xmlfile.tell(), 0) self.assertEqual(summary.isxml_stream(htmlfile), False)
class TestStreamStructure(TestInMemoryStructure): def json(self, head): return jl( jd( (q(self.LIST_TOKEN), jl(*(q(w) for w in self.LIST))), (q(self.DICT_TOKEN), jd(*((q(k), q(v)) for k,v in self.DICT.items()))), (q(self.HEAD_TOKEN), jd(*((q(k), q(v)) for k,v in self.HEAD.items()))), ), q(head), ) def get_read_store(self): self.stream = StringIO(self.json(self.HEAD_TOKEN)) return structure.JSONStreamReadStructure(self.stream) def get_write_store(self): self.stream = StringIO() return structure.JSONStreamWriteStructure(self.stream) def verify_dict(self, received, expect): tools.assert_equal(dict(expect), received) def verify_list(self, received, expect): tools.assert_equal(list(expect), received) def verify_write(self, store, head): # Nothing written to stream until close(). tools.assert_equal(0, self.stream.tell()) tools.assert_equal('', self.stream.read()) store.close() self.stream.seek(0) tools.assert_equal(self.json(head), self.stream.read()) def test_no_close(self): store = self.get_read_store() tools.assert_raises(AttributeError, getattr, store, 'close')
def open(self,f=None,mode='r',encrypt=False,sign=False): """write: encrypt = list of recipients, sign = sender or bool(default_key) read: encrypt = encrypted data expected, sign= expected key or True=defaultkey """ from six.moves import StringIO from six import next, PY3, BytesIO self.mode,self.encrypt,self.sign = mode,encrypt,sign if self.required==True: self.required = self.columns if encrypt or sign: assert self.gpg, 'gpg not intialized' self.origfile = self.file = f assert mode in ('r','w'), 'invalid mode' if mode=='r': if sign: if sign==True: fingerprint = self.gpg.default_key if type(fingerprint) == tuple: fingerprint = fingerprint[0] else: if type(sign) == tuple: sign = sign[0] fingerprint = self.gpg.find_key(sign) assert fingerprint, "sender key not found" if self.fileformat=='csv': import re if encrypt: if PY3 and isinstance(f,StringIO): result = self.gpg.decrypt_str(f.getvalue()) else: result = self.gpg.decrypt_file(f) assert result.ok, "decryption failed" if sign: assert result.valid and result.fingerprint==fingerprint, 'invalid signature' f = StringIO(str(result)) elif sign: if PY3 and isinstance(f,StringIO): result = self.gpg.verify_str(f.getvalue()) f = StringIO(self.gpg.without_signature(f.getvalue())) else: result = self.gpg.verify_file(f) f.seek(0) f = StringIO(self.gpg.without_signature(f.read())) assert result.valid and result.fingerprint==fingerprint, 'invalid signature' self.file = f dialect = self.dialect if not dialect: pos = f.tell() dialect = csv.Sniffer().sniff(f.read(1024)) f.seek(pos) # rewind reader = csv.reader(f,dialect=dialect) preamble = next(reader) assert len(preamble), 'invalid file format' assert preamble[0]==self.dataformat, "file format not supported" preamble = re.match(r'^(\d+).(\d+)',preamble[1]) assert int(preamble.group(2))<=self.version[0], "format version not supported" fields = next(reader) self.csv = reader else: # self.fileformat in ('json','jsondict','json-file','jsondict-file'): import json if self.fileformat in ('json-file','jsondict-file'): self.file = f = json.load(f) data, encrypted, signed, result = json_decrypt(f,self.gpg) assert data, 'invalid input' if encrypt: assert encrypted==bool(encrypt), 'encryption expected' if sign: assert signed==bool(sign), 'signature expected' assert result.valid and result.fingerprint==fingerprint, 'invalid signature' assert 'format' in data and data['format']==self.dataformat, "file format not supported" assert 'version' in data and data['version'][0]<=self.version[0], "file version not supported" assert 'fields' in data , "fields missing" fields = data['fields'] self.rows = data['data'] columns, unknown = [], [] for field in fields: if field in self.columns: columns.append(field) elif self.ignore: unknown.append(field) else: assert False, "unknown field '%s'" % field if self.required: for field in self.required: assert field in columns, "missing required field '%s'" % field self.fields = fields self.read_columns = (columns,unknown) elif mode=='w': assert self.fileformat in ('json','jsondict') or self.file, 'file missing' if self.fileformat=='csv': if encrypt or sign: self.file = StringIO() else: self.file = f self.csv = csv.writer(self.file,lineterminator='\n',dialect=self.dialect) self.csv.writerow((self.dataformat,'%i.%i' % tuple(self.version))) self.csv.writerow(self.columns) else: # self.fileformat in ('json','jsondict'): self.rows = []
class InputFile(object): max_buffer_size = 1024 * 1024 def __init__(self, rfile, length): """File-like object used to provide a seekable view of request body data""" self._file = rfile self.length = length self._file_position = 0 if length > self.max_buffer_size: self._buf = tempfile.TemporaryFile() else: self._buf = StringIO() @property def _buf_position(self): rv = self._buf.tell() assert rv <= self._file_position return rv def read(self, bytes=-1): assert self._buf_position <= self._file_position if bytes < 0: bytes = self.length - self._buf_position bytes_remaining = min(bytes, self.length - self._buf_position) if bytes_remaining == 0: return "" if self._buf_position != self._file_position: buf_bytes = min(bytes_remaining, self._file_position - self._buf_position) old_data = self._buf.read(buf_bytes) bytes_remaining -= buf_bytes else: old_data = "" assert bytes_remaining == 0 or self._buf_position == self._file_position, ( "Before reading buffer position (%i) didn't match file position (%i)" % (self._buf_position, self._file_position)) new_data = self._file.read(bytes_remaining) self._buf.write(new_data) self._file_position += bytes_remaining assert bytes_remaining == 0 or self._buf_position == self._file_position, ( "After reading buffer position (%i) didn't match file position (%i)" % (self._buf_position, self._file_position)) return old_data + new_data def tell(self): return self._buf_position def seek(self, offset): if offset > self.length or offset < 0: raise ValueError if offset <= self._file_position: self._buf.seek(offset) else: self.read(offset - self._file_position) def readline(self, max_bytes=None): if max_bytes is None: max_bytes = self.length - self._buf_position if self._buf_position < self._file_position: data = self._buf.readline(max_bytes) if data.endswith("\n") or len(data) == max_bytes: return data else: data = "" assert self._buf_position == self._file_position initial_position = self._file_position found = False buf = [] max_bytes -= len(data) while not found: readahead = self.read(min(2, max_bytes)) max_bytes -= len(readahead) for i, c in enumerate(readahead): if c == "\n": buf.append(readahead[:i + 1]) found = True break if not found: buf.append(readahead) if not readahead or not max_bytes: break new_data = "".join(buf) data += new_data self.seek(initial_position + len(new_data)) return data def readlines(self): rv = [] while True: data = self.readline() if data: rv.append(data) else: break return rv def next(self): data = self.readline() if data: return data else: raise StopIteration def __iter__(self): return self
def open(self,f=None,mode='r',encrypt=False,sign=False): """write: encrypt =bool or list of recipients, sign = sender or bool(default_key) read: encrypt = bool encrypted data expected, sign= expected key or True=defaultkey """ from six.moves import StringIO from six import next, PY3, BytesIO self.mode,self.encrypt,self.sign = mode,encrypt,sign if self.required is True: self.required = self.columns if encrypt or sign: assert self.gpg, 'gpg not intialized' self.origfile = self.file = f assert mode in ('r','w'), 'invalid mode' if mode=='r': if sign: if sign is True: fingerprint = self.gpg.default_key if type(fingerprint) == tuple: fingerprint = fingerprint[0] else: if type(sign) == tuple: sign = sign[0] fingerprint = self.gpg.find_key(sign) assert fingerprint, "sender key not found" if self.fileformat=='csv': import re if encrypt: if PY3 and isinstance(f,StringIO): result = self.gpg.decrypt_str(f.getvalue()) else: result = self.gpg.decrypt_file(f) assert result.ok, "decryption failed" if sign: assert result.valid and result.fingerprint==fingerprint, 'invalid signature' f = StringIO(str(result)) elif sign: if PY3 and isinstance(f,StringIO): result = self.gpg.verify_str(f.getvalue()) f = StringIO(self.gpg.without_signature(f.getvalue())) else: result = self.gpg.verify_file(f) f.seek(0) f = StringIO(self.gpg.without_signature(f.read())) assert result.valid and result.fingerprint==fingerprint, 'invalid signature' self.file = f dialect = self.dialect if not dialect: pos = f.tell() dialect = csv.Sniffer().sniff(f.read(1024)) f.seek(pos) # rewind if not PY3: import unicodecsv reader = unicodecsv.reader else: reader = csv.reader reader = reader(f,dialect=dialect) preamble = next(reader) assert len(preamble), 'invalid file format' assert preamble[0]==self.dataformat, "file format not supported" preamble = re.match(r'^(\d+).(\d+)',preamble[1]) assert int(preamble.group(2))<=self.version[0], "format version not supported" fields = next(reader) self.csv = reader else: # self.fileformat in ('json','jsondict','json-file','jsondict-file'): import json if self.fileformat in ('json-file','jsondict-file'): self.file = f = json.load(f) data, encrypted, signed, result = json_decrypt(f,self.gpg) assert data, 'invalid input' if encrypt: assert encrypted==bool(encrypt), 'encryption expected' if sign: assert signed==bool(sign), 'signature expected' assert result.valid and result.fingerprint==fingerprint, 'invalid signature' assert 'format' in data and data['format']==self.dataformat, "file format not supported" assert 'version' in data and data['version'][0]<=self.version[0], "file version not supported" assert 'fields' in data , "fields missing" fields = data['fields'] self.rows = data['data'] columns, unknown = [], [] for field in fields: if field in self.columns: columns.append(field) elif self.ignore: unknown.append(field) else: assert False, "unknown field '%s'" % field if self.required: for field in self.required: assert field in columns, "missing required field '%s'" % field self.fields = fields self.read_columns = (columns,unknown) elif mode=='w': assert self.fileformat in ('json','jsondict') or self.file, 'file missing' if self.fileformat=='csv': if encrypt or sign: self.file = StringIO() else: self.file = f if not PY3: import unicodecsv writer = unicodecsv.writer else: writer = csv.writer self.csv = writer(self.file,lineterminator='\n',dialect=self.dialect) self.csv.writerow((self.dataformat,'%i.%i' % tuple(self.version))) self.csv.writerow(self.columns) else: # self.fileformat in ('json','jsondict'): self.rows = []