def test_read_opts(): # tests if read is seeing option sets, at initialization and after # initialization arr = np.arange(6).reshape(1,6) stream = BytesIO() savemat(stream, {'a': arr}) rdr = MatFile5Reader(stream) back_dict = rdr.get_variables() rarr = back_dict['a'] assert_array_equal(rarr, arr) rdr = MatFile5Reader(stream, squeeze_me=True) assert_array_equal(rdr.get_variables()['a'], arr.reshape((6,))) rdr.squeeze_me = False assert_array_equal(rarr, arr) rdr = MatFile5Reader(stream, byte_order=boc.native_code) assert_array_equal(rdr.get_variables()['a'], arr) # inverted byte code leads to error on read because of swapped # header etc rdr = MatFile5Reader(stream, byte_order=boc.swapped_code) assert_raises(Exception, rdr.get_variables) rdr.byte_order = boc.native_code assert_array_equal(rdr.get_variables()['a'], arr) arr = np.array(['a string']) stream.truncate(0) stream.seek(0) savemat(stream, {'a': arr}) rdr = MatFile5Reader(stream) assert_array_equal(rdr.get_variables()['a'], arr) rdr = MatFile5Reader(stream, chars_as_strings=False) carr = np.atleast_2d(np.array(list(arr.item()), dtype='U1')) assert_array_equal(rdr.get_variables()['a'], carr) rdr.chars_as_strings = True assert_array_equal(rdr.get_variables()['a'], arr)
def tar_stream(store, tree, mtime, format=''): """Generate a tar stream for the contents of a Git tree. Returns a generator that lazily assembles a .tar.gz archive, yielding it in pieces (bytestrings). To obtain the complete .tar.gz binary file, simply concatenate these chunks. :param store: Object store to retrieve objects from :param tree: Tree object for the tree root :param mtime: UNIX timestamp that is assigned as the modification time for all files :param format: Optional compression format for tarball :return: Bytestrings """ buf = BytesIO() with closing(tarfile.open(None, "w:%s" % format, buf)) as tar: for entry_abspath, entry in _walk_tree(store, tree): try: blob = store[entry.sha] except KeyError: # Entry probably refers to a submodule, which we don't yet support. continue data = ChunkedBytesIO(blob.chunked) info = tarfile.TarInfo() info.name = entry_abspath.decode('ascii') # tarfile only works with ascii. info.size = blob.raw_length() info.mode = entry.mode info.mtime = mtime tar.addfile(info, data) yield buf.getvalue() buf.truncate(0) buf.seek(0) yield buf.getvalue()
def test_empty_string(): # make sure reading empty string does not raise error estring_fname = pjoin(test_data_path, 'single_empty_string.mat') fp = open(estring_fname, 'rb') rdr = MatFile5Reader(fp) d = rdr.get_variables() fp.close() assert_array_equal(d['a'], np.array([], dtype='U1')) # empty string round trip. Matlab cannot distiguish # between a string array that is empty, and a string array # containing a single empty string, because it stores strings as # arrays of char. There is no way of having an array of char that # is not empty, but contains an empty string. stream = BytesIO() savemat(stream, {'a': np.array([''])}) rdr = MatFile5Reader(stream) d = rdr.get_variables() assert_array_equal(d['a'], np.array([], dtype='U1')) stream.truncate(0) stream.seek(0) savemat(stream, {'a': np.array([], dtype='U1')}) rdr = MatFile5Reader(stream) d = rdr.get_variables() assert_array_equal(d['a'], np.array([], dtype='U1')) stream.close()
class CairoContext: def __init__(self, width, height): self.outbuf = BytesIO() self.surface = cairo.PDFSurface(self.outbuf, width, height) self.context = cairo.Context(self.surface) def set_size(self, w, h): self.surface.set_size(w, h) def render(self, svg): if not svg.render_cairo(self.context): log.critical("SVG failed to render") else: # flush the page to the drawing surface self.surface.show_page() def finish(self): "Closes the drawing surface and pushes the last data out" self.surface.finish() return self.flush() def flush(self): "Returns data from the buffer and resets buffer to 0" out = self.outbuf.getvalue() self.outbuf.seek(0) self.outbuf.truncate(0) return out
class TruncatedTailPipe(object): """ Truncate the last `tail_size` bytes from the stream. """ def __init__(self, output=None, tail_size=16): self.tail_size = tail_size self.output = output or BytesIO() self.buffer = BytesIO() def write(self, data): self.buffer.write(data) if self.buffer.tell() > self.tail_size: self._truncate_tail() def _truncate_tail(self): overflow_size = self.buffer.tell() - self.tail_size self.buffer.seek(0) self.output.write(self.buffer.read(overflow_size)) remaining = self.buffer.read() self.buffer.seek(0) self.buffer.write(remaining) self.buffer.truncate() def close(self): return self.output
class UnicodeWriter(object): """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, dialect=csv.excel_tab, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = BytesIO() if six.PY2 else StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) def _encode(self, item): if six.PY2: encoded = unicode(item).encode('utf-8') else: encoded = str(item) return encoded def writerow(self, row): self.writer.writerow([self._encode(s) for s in row]) def get_values(self): # Fetch UTF-8 output from the queue ... ret = self.queue.getvalue() # empty queue self.queue.truncate(0) if six.PY2: return ret.lstrip(b'\0') return ret.encode('utf-8').lstrip(b'\0') def writerows(self, rows): for row in rows: self.writerow(row)
class StreamCapturer(Thread): started = False def __init__(self): super(StreamCapturer, self).__init__() self.streams = [] self.buffer = BytesIO() self.streams_lock = Lock() self.buffer_lock = Lock() self.stream_added = Event() self.stop = Event() def run(self): self.started = True while not self.stop.is_set(): with self.streams_lock: streams = self.streams if not streams: self.stream_added.wait(timeout=1) self.stream_added.clear() continue ready = select(streams, [], [], 0.5)[0] dead = [] with self.buffer_lock: for fd in ready: try: self.buffer.write(os.read(fd, 1024)) except OSError as e: import errno if e.errno == errno.EBADF: dead.append(fd) else: raise with self.streams_lock: for fd in dead: self.streams.remove(fd) def add_stream(self, fd): with self.streams_lock: self.streams.append(fd) self.stream_added.set() def remove_stream(self, fd): with self.streams_lock: self.streams.remove(fd) def reset_buffer(self): with self.buffer_lock: self.buffer.truncate(0) self.buffer.seek(0) def get_buffer(self): with self.buffer_lock: return self.buffer.getvalue() def ensure_started(self): if not self.started: self.start()
def generator(): # python2 needs byes when python3 needs unicode if six.PY2: stringio = BytesIO() else: stringio = StringIO() csv_writer = csv.writer(stringio) it = iter(resultdb.select(project)) first_30 = [] for result in it: first_30.append(result) if len(first_30) >= 30: break common_fields, _ = result_formater(first_30) common_fields_l = sorted(common_fields) csv_writer.writerow([toString('url')] + [toString(x) for x in common_fields_l] + [toString('...')]) for result in itertools.chain(first_30, it): other = {} for k, v in iteritems(result['result']): if k not in common_fields: other[k] = v csv_writer.writerow( [toString(result['url'])] + [toString(result['result'].get(k, '')) for k in common_fields_l] + [toString(other)] ) yield stringio.getvalue() stringio.truncate(0)
class ChunkBuffer: BUFFER_SIZE = 1 * 1024 * 1024 def __init__(self, key, chunker_params=CHUNKER_PARAMS): self.buffer = BytesIO() self.packer = msgpack.Packer(unicode_errors='surrogateescape') self.chunks = [] self.key = key self.chunker = Chunker(self.key.chunk_seed, *chunker_params) def add(self, item): self.buffer.write(self.packer.pack(StableDict(item))) if self.is_full(): self.flush() def write_chunk(self, chunk): raise NotImplementedError def flush(self, flush=False): if self.buffer.tell() == 0: return self.buffer.seek(0) chunks = list(bytes(s) for s in self.chunker.chunkify(self.buffer)) self.buffer.seek(0) self.buffer.truncate(0) # Leave the last partial chunk in the buffer unless flush is True end = None if flush or len(chunks) == 1 else -1 for chunk in chunks[:end]: self.chunks.append(self.write_chunk(chunk)) if end == -1: self.buffer.write(chunks[-1]) def is_full(self): return self.buffer.tell() > self.BUFFER_SIZE
class CsvDictsAdapter(object): """Provide a DataChange generator and it provides a file-like object which returns csv data""" def __init__(self, source_generator): self.source = source_generator self.buffer = BytesIO() self.csv = None self.add_header = False def __iter__(self): return self def write_header(self): self.add_header = True def next(self): row = self.source.next() self.buffer.truncate(0) self.buffer.seek(0) if not self.csv: self.csv = csv.DictWriter(self.buffer, row.keys(), quoting=csv.QUOTE_NONNUMERIC, encoding='utf-8') self.add_header = True if self.add_header: if hasattr(self.csv, 'writeheader'): self.csv.writeheader() else: self.csv.writerow(dict((fn, fn) for fn in self.csv.fieldnames)) self.add_header = False self.csv.writerow(row) self.buffer.seek(0) return self.buffer.read()
def test_str_round(): # from report by Angus McMorland on mailing list 3 May 2010 stream = BytesIO() in_arr = np.array(['Hello', 'Foob']) out_arr = np.array(['Hello', 'Foob ']) savemat(stream, dict(a=in_arr)) res = loadmat(stream) # resulted in ['HloolFoa', 'elWrdobr'] assert_array_equal(res['a'], out_arr) stream.truncate(0) stream.seek(0) # Make Fortran ordered version of string in_str = in_arr.tostring(order='F') in_from_str = np.ndarray(shape=a.shape, dtype=in_arr.dtype, order='F', buffer=in_str) savemat(stream, dict(a=in_from_str)) assert_array_equal(res['a'], out_arr) # unicode save did lead to buffer too small error stream.truncate(0) stream.seek(0) in_arr_u = in_arr.astype('U') out_arr_u = out_arr.astype('U') savemat(stream, {'a': in_arr_u}) res = loadmat(stream) assert_array_equal(res['a'], out_arr_u)
class UnicodeWriter: """ CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. We are using this custom UnicodeWriter for python versions 2.x """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): self.queue = BytesIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() def writerow(self, row): self.writer.writerow([s.encode("utf-8") for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for row in rows: self.writerow(row)
def test_array_file_scales(): # Test scaling works for max, min when going from larger to smaller type, # and from float to integer. bio = BytesIO() for in_type, out_type, err in ((np.int16, np.int16, None), (np.int16, np.int8, None), (np.uint16, np.uint8, None), (np.int32, np.int8, None), (np.float32, np.uint8, None), (np.float32, np.int16, None)): out_dtype = np.dtype(out_type) arr = np.zeros((3,), dtype=in_type) info = type_info(in_type) arr[0], arr[1] = info['min'], info['max'] if not err is None: assert_raises(err, calculate_scale, arr, out_dtype, True) continue slope, inter, mn, mx = calculate_scale(arr, out_dtype, True) array_to_file(arr, bio, out_type, 0, inter, slope, mn, mx) bio.seek(0) arr2 = array_from_file(arr.shape, out_dtype, bio) arr3 = apply_read_scaling(arr2, slope, inter) # Max rounding error for integer type max_miss = slope / 2. assert_true(np.all(np.abs(arr - arr3) <= max_miss)) bio.truncate(0) bio.seek(0)
def tar_stream(repo, tree, mtime, format=''): """ Returns a generator that lazily assembles a .tar.gz archive, yielding it in pieces (bytestrings). To obtain the complete .tar.gz binary file, simply concatenate these chunks. 'repo' and 'tree' are the dulwich Repo and Tree objects the archive shall be created from. 'mtime' is a UNIX timestamp that is assigned as the modification time of all files in the resulting .tar.gz archive. """ buf = BytesIO() with closing(tarfile.open(None, "w:%s" % format, buf)) as tar: for entry_abspath, entry in walk_tree(repo, tree): try: blob = repo[entry.sha] except KeyError: # Entry probably refers to a submodule, which we don't yet support. continue data = ListBytesIO(blob.chunked) info = tarfile.TarInfo() info.name = entry_abspath info.size = blob.raw_length() info.mode = entry.mode info.mtime = mtime tar.addfile(info, data) yield buf.getvalue() buf.truncate(0) buf.seek(0) yield buf.getvalue()
def _download_part( self, part_num ): """ Download a part from the source URL. Returns a BytesIO buffer. The buffer's tell() method will return the size of the downloaded part, which may be less than the requested part size if the part is the last one for the URL. """ buf = BytesIO( ) with closing( pycurl.Curl( ) ) as c: c.setopt( c.URL, self.url ) c.setopt( c.WRITEDATA, buf ) c.setopt( c.FAILONERROR, 1 ) start, end = self._get_part_range( part_num ) c.setopt( c.RANGE, "%i-%i" % (start, end - 1) ) try: c.perform( ) except pycurl.error as e: error_code, message = e if error_code == c.E_BAD_DOWNLOAD_RESUME: # bad range for FTP pass elif error_code == c.E_HTTP_RETURNED_ERROR: # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.17 if c.getinfo( c.RESPONSE_CODE ) == 416: pass else: raise else: raise buf.truncate( 0 ) buf.seek( 0 ) return buf
class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = BytesIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)('replace') def writerow(self, row): row = [smart_text(s) for s in row] self.writer.writerow([s.encode("utf-8") for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for row in rows: self.writerow(row)
class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() def writerow(self, row): try: self.writer.writerow([s.decode('utf8').encode("utf-8") if isinstance(s, basestring) else s for s in row]) except: self.writer.writerow([s.encode("utf-8") if isinstance(s, basestring) else s for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for row in rows: self.writerow(row)
class StatefulCommandCompiler(CommandCompiler): """A command compiler that buffers input until a full command is available.""" def __init__(self): super().__init__() self.buf = BytesIO() def is_partial_command(self): return bool(self.buf.getvalue()) def __call__(self, source, **kwargs): buf = self.buf if self.is_partial_command(): buf.write(b'\n') buf.write(source) code = self.buf.getvalue().decode('utf8') codeobj = super().__call__(code, **kwargs) if codeobj: self.reset() return codeobj def reset(self): self.buf.seek(0) self.buf.truncate(0)
def push(): # set this to Application ID from Instapush appID = "<FILL THIS IN>" # set this to the Application Secret from Instapush appSecret = "<FILL THIS IN>" # leave this set to DoorAlert unless you named your event something different in Instapush pushEvent = "Doorbell" # set this to what you want the push message to say pushMessage = "Ding dong! Check doorbell.jpg on dropbox!" # use StringIO to capture the response from our push API call buffer = StringIO() # use Curl to post to the Instapush API c = pycurl.Curl() # set Instapush API URL c.setopt(c.URL, 'https://api.instapush.im/v1/post') # setup custom headers for authentication variables and content type c.setopt(c.HTTPHEADER, ['x-instapush-appid: ' + appID, 'x-instapush-appsecret: ' + appSecret, 'Content-Type: application/json']) # create a dictionary structure for the JSON data to post to Instapush json_fields = {} # setup JSON values json_fields['event'] = pushEvent json_fields['trackers'] = {} json_fields['trackers']['msg'] = pushMessage postfields = json.dumps(json_fields) # make sure to send the JSON with post c.setopt(c.POSTFIELDS, postfields) # set this so we can capture the resposne in our buffer c.setopt(c.WRITEFUNCTION, buffer.write) # uncomment to see the post that is sent # c.setopt(c.VERBOSE, True) c.perform() # capture the response from the server body = buffer.getvalue() # print the response print(body) # reset the buffer buffer.truncate(0) buffer.seek(0) # cleanup c.close()
class ChunkedReader(object): def __init__(self, socket_file, maxsize): self.socket_file, self.maxsize = socket_file, maxsize self.rbuf = BytesIO() self.bytes_read = 0 self.finished = False def check_size(self): if self.bytes_read > self.maxsize: raise MaxSizeExceeded('Request entity too large', self.bytes_read, self.maxsize) def read_chunk(self): if self.finished: return line = self.socket_file.readline() self.bytes_read += len(line) self.check_size() chunk_size = line.strip().split(b';', 1)[0] try: chunk_size = int(line, 16) + 2 except Exception: raise ValueError('%s is not a valid chunk size' % reprlib.repr(chunk_size)) if chunk_size + self.bytes_read > self.maxsize: raise MaxSizeExceeded('Request entity too large', self.bytes_read + chunk_size, self.maxsize) chunk = self.socket_file.read(chunk_size) if len(chunk) < chunk_size: raise ValueError('Bad chunked encoding, chunk truncated: %d < %s' % (len(chunk), chunk_size)) if not chunk.endswith(b'\r\n'): raise ValueError('Bad chunked encoding: %r != CRLF' % chunk[:-2]) self.rbuf.seek(0, os.SEEK_END) self.bytes_read += chunk_size if chunk_size == 2: self.finished = True else: self.rbuf.write(chunk[:-2]) def read(self, size=-1): if size < 0: # Read all data while not self.finished: self.read_chunk() self.rbuf.seek(0) rv = self.rbuf.read() if rv: self.rbuf.truncate(0) return rv if size == 0: return b'' while self.rbuf.tell() < size and not self.finished: self.read_chunk() data = self.rbuf.getvalue() self.rbuf.truncate(0) if size < len(data): self.rbuf.write(data[size:]) return data[:size] return data
class Serialization(unittest.TestCase): def setUp(self): self.t1 = m.SporadicTask(10, 100) self.t2 = m.SporadicTask(5, 19, 15, id=3) self.t3 = m.SporadicTask(25, 50, id=5, deadline=75) self.ts = m.TaskSystem([self.t1, self.t2, self.t3]) self.f = BytesIO() def test_serialize_task(self): for t in self.ts: s.write_xml(s.task(t), self.f) self.f.seek(0) x = s.load(self.f) self.assertIsInstance(x, m.SporadicTask) self.assertEqual(x.cost, t.cost) self.assertEqual(x.deadline, t.deadline) self.assertEqual(x.period, t.period) self.assertEqual(x.id, t.id) self.f.seek(0) self.f.truncate() def test_serialize_taskset(self): s.write(self.ts, self.f) self.f.seek(0) xs = s.load(self.f) self.assertIsInstance(xs, m.TaskSystem) self.assertEqual(len(xs), len(self.ts)) for x,t in zip(xs, self.ts): self.assertEqual(x.cost, t.cost) self.assertEqual(x.deadline, t.deadline) self.assertEqual(x.period, t.period) self.assertEqual(x.id, t.id) def test_serialize_resmodel(self): r.initialize_resource_model(self.ts) self.t1.resmodel[1].add_request(1) self.t2.resmodel[1].add_read_request(2) self.t2.resmodel['serial I/O'].add_request(2) self.t3.resmodel['serial I/O'].add_request(3) for t in self.ts: s.write_xml(s.task(t), self.f) self.f.seek(0) x = s.load(self.f) self.assertIsInstance(x.resmodel, r.ResourceRequirements) self.assertEqual(len(x.resmodel), len(t.resmodel)) self.assertEqual(list(x.resmodel.keys()), list(t.resmodel.keys())) for res_id in x.resmodel: self.assertEqual(x.resmodel[res_id].max_reads, t.resmodel[res_id].max_reads) self.assertEqual(x.resmodel[res_id].max_writes, t.resmodel[res_id].max_writes) self.assertEqual(x.resmodel[res_id].max_requests, t.resmodel[res_id].max_requests) self.assertEqual(x.resmodel[res_id].max_read_length, t.resmodel[res_id].max_read_length) self.assertEqual(x.resmodel[res_id].max_write_length, t.resmodel[res_id].max_write_length) self.assertEqual(x.resmodel[res_id].max_length, t.resmodel[res_id].max_length) self.f.seek(0) self.f.truncate()
def test_round_types(): # Check that saving, loading preserves dtype in most cases arr = np.arange(10) stream = BytesIO() for dts in ("f8", "f4", "i8", "i4", "i2", "i1", "u8", "u4", "u2", "u1", "c16", "c8"): stream.truncate(0) stream.seek(0) # needed for BytesIO in python 3 savemat_future(stream, {"arr": arr.astype(dts)}) vars = loadmat(stream) assert_equal(np.dtype(dts), vars["arr"].dtype)
def alpino_parse_tokenized_file_socket(tokenized_file_path: Path, target_dir_path: Path, host='127.0.0.1', base_port_number=42424, n_instances: int=4): """ Warning: produces corrupt output for half of the sentences. """ chosen_port = randrange(base_port_number, base_port_number + n_instances) parsed_sentences_files_dir_path, parsed_sentence_file_path_prefix = \ prepare_parsing(tokenized_file_path=tokenized_file_path, target_dir_path=target_dir_path) if parsed_sentences_files_dir_path is not None: with tokenized_file_path.open(mode='rb') as tokenized_file: parsed_sentence_buffer = BytesIO() sentence_index = 0 for sentence in tokenized_file.readlines(): sentence_index += 1 with socket(AF_INET, SOCK_STREAM) as alpino_socket: # alpino_socket.settimeout() # TODO: set timeout equal # to Alpino timeout alpino_socket.connect_ex((host, chosen_port)) alpino_socket.sendall(sentence + b'\n\n') while True: parsed_sentence_xml_chunk = alpino_socket.recv( ALPINO_SOCKET_BUFFER_SIZE) if not parsed_sentence_xml_chunk: alpino_socket.sendall(b'\n\n') break else: parsed_sentence_buffer.write( parsed_sentence_xml_chunk) parsed_sentence = parsed_sentence_buffer.getvalue() parsed_sentence_buffer.truncate() parsed_sentence_buffer.flush() parsed_sentence_buffer.seek(0) parsed_sentence_file_path = \ parsed_sentence_file_path_prefix.with_suffix( '.{0:d}.xml'.format(sentence_index)) with parsed_sentence_file_path.open( mode='wb') as parsed_sentence_file: parsed_sentence_file.write(parsed_sentence) info( "Parsed tokenized file to '{" "parsed_sentence_file_path_prefix}'.*.xml . " "".format( parsed_sentence_file_path_prefix=parsed_sentence_file_path_prefix))
def test_round_types(): # Check that saving, loading preserves dtype in most cases arr = np.arange(10) stream = BytesIO() for dts in ('f8','f4','i8','i4','i2','i1', 'u8','u4','u2','u1','c16','c8'): stream.truncate(0) stream.seek(0) # needed for BytesIO in python 3 savemat(stream, {'arr': arr.astype(dts)}) vars = loadmat(stream) assert_equal(np.dtype(dts), vars['arr'].dtype)
def sio_func(): fio = BytesIO() fio.truncate(0) fio.seek(offset) fio.write(data) # Use a copy of the header to avoid changing # global header in test functions. new_hdr = hdr.copy() return (self.proxy_class(fio, new_hdr), fio, new_hdr)
def test_digest(self): iio = BytesIO() for tv in self.__class__.TEST_VECTORS: iio.truncate(0) iio.seek(0) iio.write(tv.text) iio.seek(0) digest = tv.cls.digest(iio) self.assertEqual(tv.digest, digest, "{}{}".format(tv.cls, tv.text))
def thumbnail(image_data: io.BytesIO, max_size=(128, 128)) -> None: """Resize an image in place to no more than max_size pixels, preserving aspect ratio.""" with wand.image.Image(blob=image_data) as image: new_resolution = scale_resolution((image.width, image.height), max_size) image.resize(*new_resolution) image_data.truncate(0) image_data.seek(0) image.save(file=image_data) # allow resizing the original image more than once for memory profiling image_data.seek(0)
def droptests(fp, **kw): """ Drop tests from chrome.manifest """ kw = kw io = BytesIO() with Reset(io): for l in fp: if b"dta-tests" in l: continue io.write(l) io.truncate() return io
class TestQuotedPrintableDecoder(unittest.TestCase): def setUp(self): self.f = BytesIO() self.d = QuotedPrintableDecoder(self.f) def assert_data(self, data, finalize=True): if finalize: self.d.finalize() self.f.seek(0) self.assertEqual(self.f.read(), data) self.f.seek(0) self.f.truncate() def test_simple(self): self.d.write(b'foobar') self.assert_data(b'foobar') def test_with_escape(self): self.d.write(b'foo=3Dbar') self.assert_data(b'foo=bar') def test_with_newline_escape(self): self.d.write(b'foo=\r\nbar') self.assert_data(b'foobar') def test_with_only_newline_escape(self): self.d.write(b'foo=\nbar') self.assert_data(b'foobar') def test_with_split_escape(self): self.d.write(b'foo=3') self.d.write(b'Dbar') self.assert_data(b'foo=bar') def test_with_split_newline_escape_1(self): self.d.write(b'foo=\r') self.d.write(b'\nbar') self.assert_data(b'foobar') def test_with_split_newline_escape_2(self): self.d.write(b'foo=') self.d.write(b'\r\nbar') self.assert_data(b'foobar') def test_close_and_finalize(self): parser = Mock() f = QuotedPrintableDecoder(parser) f.finalize() parser.finalize.assert_called_once_with() f.close() parser.close.assert_called_once_with()
class BufferedPktLineWriterTests(TestCase): def setUp(self): TestCase.setUp(self) self._output = BytesIO() self._writer = BufferedPktLineWriter(self._output.write, bufsize=16) def assertOutputEquals(self, expected): self.assertEqual(expected, self._output.getvalue()) def _truncate(self): self._output.seek(0) self._output.truncate() def test_write(self): self._writer.write(b'foo') self.assertOutputEquals(b'') self._writer.flush() self.assertOutputEquals(b'0007foo') def test_write_none(self): self._writer.write(None) self.assertOutputEquals(b'') self._writer.flush() self.assertOutputEquals(b'0000') def test_flush_empty(self): self._writer.flush() self.assertOutputEquals(b'') def test_write_multiple(self): self._writer.write(b'foo') self._writer.write(b'bar') self.assertOutputEquals(b'') self._writer.flush() self.assertOutputEquals(b'0007foo0007bar') def test_write_across_boundary(self): self._writer.write(b'foo') self._writer.write(b'barbaz') self.assertOutputEquals(b'0007foo000abarba') self._truncate() self._writer.flush() self.assertOutputEquals(b'z') def test_write_to_boundary(self): self._writer.write(b'foo') self._writer.write(b'barba') self.assertOutputEquals(b'0007foo0009barba') self._truncate() self._writer.write(b'z') self._writer.flush() self.assertOutputEquals(b'0005z')
class DummyResource(AbstractResource): _root = None def __init__(self, prefix: str, path: str = '/', is_collection=True, parent=None, ctime=None, mtime=None, exists=None): super().__init__(prefix, path) if path == '/': self._exists = True if self._root: raise ValueError("Second _root") self.__class__._root = self else: self._exists = exists self._is_directory = is_collection if self._exists: if not is_collection: self._content = BytesIO() else: self._resources = {} dirname = os.path.dirname(path) self._parent = parent or self._root.with_relative(dirname) self._ctime = ctime or datetime.now() self._mtime = mtime or datetime.now() def _touch_file(self): # noinspection PyProtectedMember if not self.parent or not self.parent._exists: raise errors.ResourceDoesNotExist("Parent resource does not exist") if not self.parent.is_collection: raise errors.InvalidResourceType("Collection expected") # noinspection PyProtectedMember self._parent._resources[self.name] = self self._exists = True self._content = BytesIO() self._is_directory = False async def get_content(self, write: typing.Callable[[bytes], typing.Any], *, offset: int = 0, limit: int = None): if self._is_directory: raise errors.InvalidResourceType("file resource expected") self._content.seek(offset) buffer = self._content.read(limit) if asyncio.iscoroutinefunction(write): await write(buffer) else: write(buffer) def with_relative(self, relative) -> 'AbstractResource': if relative == '/': return self res = self parts = relative.strip('/').split('/') for part in parts[:-1]: try: res = res._resources[part] except KeyError: raise errors.ResourceDoesNotExist( "one of parent resources does not exist") part = parts[-1] try: if not res.is_collection: raise errors.InvalidResourceType("collection expected") res = res._resources[part] except KeyError: res = DummyResource(self.prefix, os.path.join(res.path, part), parent=res, exists=False) return res @property def size(self) -> int: if self.is_collection: return 0 return len(self._content.getvalue()) @property def mtime(self): return self._mtime @property def ctime(self): return self._ctime async def put_content(self, read_some: typing.Awaitable[bytes]) -> bool: if self._exists: created = False if self.is_collection: raise errors.InvalidResourceType("file resource expected") else: created = True self._touch_file() self._content.seek(0) self._content.truncate() if not read_some: return created while True: buffer = await read_some() self._content.write(buffer) if not buffer: return created def propfind(self, *props) -> OrderedDict: fmt = '%Y-%m-%dT%H:%M:%SZ' ctime = self._ctime.strftime(fmt) mtime = self._mtime.strftime(fmt) all_props = OrderedDict([ ('getcontenttype', ''), ('getlastmodified', mtime), ('getcontentlength', self.size), ('getetag', ''), ('creationdate', ctime), ('displayname', self.name), ]) if not props: return all_props return OrderedDict(p for p in all_props.items() if p[0] in props) async def populate_props(self): if not self._exists: raise errors.ResourceDoesNotExist() return async def populate_collection(self): if not self._exists: raise errors.ResourceDoesNotExist() return @property def parent(self) -> 'AbstractResource': if self._path == '/': return None return self._parent @property def name(self) -> str: return os.path.basename(self.path) # noinspection PyProtectedMember async def move(self, destination: str) -> bool: old_name = self.name dest_resource = self._root.with_relative(destination) if not dest_resource._exists: self._path = dest_resource.path dest_resource = dest_resource._parent await dest_resource.populate_props() if not dest_resource.is_collection: raise errors.InvalidResourceType("collection expected") del self._parent._resources[old_name] self._parent = dest_resource self._path = os.path.join(self._parent.path, self.name) dest_resource._resources[self.name] = self return True async def make_collection(self, collection: str) -> 'AbstractResource': collection = collection.strip('/') parent = os.path.dirname(collection) name = os.path.basename(collection) if not parent: parent = self else: parent = self._root.with_relative(parent) if not parent.is_collection: raise errors.InvalidResourceType("collection expected") # noinspection PyProtectedMember if name in parent._resources: raise errors.ResourceAlreadyExists("collection already exists") collection = DummyResource(prefix=self.prefix, path=os.path.join(parent.path, collection), parent=parent, is_collection=True, exists=True) # noinspection PyProtectedMember parent._resources[name] = collection return collection @property def is_collection(self): return self._is_directory async def delete(self): if not self._exists: raise errors.ResourceDoesNotExist() # noinspection PyProtectedMember del self._parent._resources[self.name] # noinspection PyProtectedMember async def copy(self, destination: str) -> 'AbstractResource': if not self.is_collection: return self._copy_file(destination) dest = self._root / destination.strip('/') if not dest.is_collection: raise errors.InvalidResourceType("collection expected") if not dest._exists: name = dest.name dest = dest._parent else: name = self.name new_dir = self._clone() new_dir._parent = dest dest._resources[name] = new_dir new_dir._path = os.path.join(dest.path, name) for res in self.collection: await res.copy(new_dir.path) return new_dir # noinspection PyProtectedMember def _copy_file(self, destination: str) -> 'AbstractResource': dest = self._root / destination.strip('/') if dest._exists: if not dest.is_collection: raise errors.ResourceAlreadyExists( "destination file already exists") parent = dest name = self.name else: parent = dest._parent name = dest.name new_file = self._clone() new_file._parent = parent new_file._path = os.path.join(parent.path, name) parent._resources[name] = new_file return new_file @property def collection(self) -> typing.List['AbstractResource']: result = [] for k in sorted(self._resources.keys()): result.append(self._resources[k]) return result def _clone(self): new = DummyResource(prefix=self.prefix, path=self.path, is_collection=self.is_collection, parent=self._parent, exists=self._exists) if new.is_collection: new._resources = {} else: new._content = BytesIO(self._content.getvalue()) return new def __repr__(self): return "Dummy<%s>" % self.path # pragma: no cover
class NetstringReceiver(protocol.Protocol): """ A protocol that sends and receives netstrings. See U{http://cr.yp.to/proto/netstrings.txt} for the specification of netstrings. Every netstring starts with digits that specify the length of the data. This length specification is separated from the data by a colon. The data is terminated with a comma. Override L{stringReceived} to handle received netstrings. This method is called with the netstring payload as a single argument whenever a complete netstring is received. Security features: 1. Messages are limited in size, useful if you don't want someone sending you a 500MB netstring (change C{self.MAX_LENGTH} to the maximum length you wish to accept). 2. The connection is lost if an illegal message is received. @ivar MAX_LENGTH: Defines the maximum length of netstrings that can be received. @type MAX_LENGTH: C{int} @ivar _LENGTH: A pattern describing all strings that contain a netstring length specification. Examples for length specifications are C{b'0:'}, C{b'12:'}, and C{b'179:'}. C{b'007:'} is not a valid length specification, since leading zeros are not allowed. @type _LENGTH: C{re.Match} @ivar _LENGTH_PREFIX: A pattern describing all strings that contain the first part of a netstring length specification (without the trailing comma). Examples are '0', '12', and '179'. '007' does not start a netstring length specification, since leading zeros are not allowed. @type _LENGTH_PREFIX: C{re.Match} @ivar _PARSING_LENGTH: Indicates that the C{NetstringReceiver} is in the state of parsing the length portion of a netstring. @type _PARSING_LENGTH: C{int} @ivar _PARSING_PAYLOAD: Indicates that the C{NetstringReceiver} is in the state of parsing the payload portion (data and trailing comma) of a netstring. @type _PARSING_PAYLOAD: C{int} @ivar brokenPeer: Indicates if the connection is still functional @type brokenPeer: C{int} @ivar _state: Indicates if the protocol is consuming the length portion (C{PARSING_LENGTH}) or the payload (C{PARSING_PAYLOAD}) of a netstring @type _state: C{int} @ivar _remainingData: Holds the chunk of data that has not yet been consumed @type _remainingData: C{string} @ivar _payload: Holds the payload portion of a netstring including the trailing comma @type _payload: C{BytesIO} @ivar _expectedPayloadSize: Holds the payload size plus one for the trailing comma. @type _expectedPayloadSize: C{int} """ MAX_LENGTH = 99999 _LENGTH = re.compile(b'(0|[1-9]\d*)(:)') _LENGTH_PREFIX = re.compile(b'(0|[1-9]\d*)$') # Some error information for NetstringParseError instances. _MISSING_LENGTH = ("The received netstring does not start with a " "length specification.") _OVERFLOW = ("The length specification of the received netstring " "cannot be represented in Python - it causes an " "OverflowError!") _TOO_LONG = ("The received netstring is longer than the maximum %s " "specified by self.MAX_LENGTH") _MISSING_COMMA = "The received netstring is not terminated by a comma." # The following constants are used for determining if the NetstringReceiver # is parsing the length portion of a netstring, or the payload. _PARSING_LENGTH, _PARSING_PAYLOAD = range(2) def makeConnection(self, transport): """ Initializes the protocol. """ protocol.Protocol.makeConnection(self, transport) self._remainingData = b"" self._currentPayloadSize = 0 self._payload = BytesIO() self._state = self._PARSING_LENGTH self._expectedPayloadSize = 0 self.brokenPeer = 0 def sendString(self, string): """ Sends a netstring. Wraps up C{string} by adding length information and a trailing comma; writes the result to the transport. @param string: The string to send. The necessary framing (length prefix, etc) will be added. @type string: C{bytes} """ self.transport.write(_formatNetstring(string)) def dataReceived(self, data): """ Receives some characters of a netstring. Whenever a complete netstring is received, this method extracts its payload and calls L{stringReceived} to process it. @param data: A chunk of data representing a (possibly partial) netstring @type data: C{bytes} """ self._remainingData += data while self._remainingData: try: self._consumeData() except IncompleteNetstring: break except NetstringParseError: self._handleParseError() break def stringReceived(self, string): """ Override this for notification when each complete string is received. @param string: The complete string which was received with all framing (length prefix, etc) removed. @type string: C{bytes} @raise NotImplementedError: because the method has to be implemented by the child class. """ raise NotImplementedError() def _maxLengthSize(self): """ Calculate and return the string size of C{self.MAX_LENGTH}. @return: The size of the string representation for C{self.MAX_LENGTH} @rtype: C{float} """ return math.ceil(math.log10(self.MAX_LENGTH)) + 1 def _consumeData(self): """ Consumes the content of C{self._remainingData}. @raise IncompleteNetstring: if C{self._remainingData} does not contain enough data to complete the current netstring. @raise NetstringParseError: if the received data do not form a valid netstring. """ if self._state == self._PARSING_LENGTH: self._consumeLength() self._prepareForPayloadConsumption() if self._state == self._PARSING_PAYLOAD: self._consumePayload() def _consumeLength(self): """ Consumes the length portion of C{self._remainingData}. @raise IncompleteNetstring: if C{self._remainingData} contains a partial length specification (digits without trailing comma). @raise NetstringParseError: if the received data do not form a valid netstring. """ lengthMatch = self._LENGTH.match(self._remainingData) if not lengthMatch: self._checkPartialLengthSpecification() raise IncompleteNetstring() self._processLength(lengthMatch) def _checkPartialLengthSpecification(self): """ Makes sure that the received data represents a valid number. Checks if C{self._remainingData} represents a number smaller or equal to C{self.MAX_LENGTH}. @raise NetstringParseError: if C{self._remainingData} is no number or is too big (checked by L{_extractLength}). """ partialLengthMatch = self._LENGTH_PREFIX.match(self._remainingData) if not partialLengthMatch: raise NetstringParseError(self._MISSING_LENGTH) lengthSpecification = (partialLengthMatch.group(1)) self._extractLength(lengthSpecification) def _processLength(self, lengthMatch): """ Processes the length definition of a netstring. Extracts and stores in C{self._expectedPayloadSize} the number representing the netstring size. Removes the prefix representing the length specification from C{self._remainingData}. @raise NetstringParseError: if the received netstring does not start with a number or the number is bigger than C{self.MAX_LENGTH}. @param lengthMatch: A regular expression match object matching a netstring length specification @type lengthMatch: C{re.Match} """ endOfNumber = lengthMatch.end(1) startOfData = lengthMatch.end(2) lengthString = self._remainingData[:endOfNumber] # Expect payload plus trailing comma: self._expectedPayloadSize = self._extractLength(lengthString) + 1 self._remainingData = self._remainingData[startOfData:] def _extractLength(self, lengthAsString): """ Attempts to extract the length information of a netstring. @raise NetstringParseError: if the number is bigger than C{self.MAX_LENGTH}. @param lengthAsString: A chunk of data starting with a length specification @type lengthAsString: C{bytes} @return: The length of the netstring @rtype: C{int} """ self._checkStringSize(lengthAsString) length = int(lengthAsString) if length > self.MAX_LENGTH: raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH, )) return length def _checkStringSize(self, lengthAsString): """ Checks the sanity of lengthAsString. Checks if the size of the length specification exceeds the size of the string representing self.MAX_LENGTH. If this is not the case, the number represented by lengthAsString is certainly bigger than self.MAX_LENGTH, and a NetstringParseError can be raised. This method should make sure that netstrings with extremely long length specifications are refused before even attempting to convert them to an integer (which might trigger a MemoryError). """ if len(lengthAsString) > self._maxLengthSize(): raise NetstringParseError(self._TOO_LONG % (self.MAX_LENGTH, )) def _prepareForPayloadConsumption(self): """ Sets up variables necessary for consuming the payload of a netstring. """ self._state = self._PARSING_PAYLOAD self._currentPayloadSize = 0 self._payload.seek(0) self._payload.truncate() def _consumePayload(self): """ Consumes the payload portion of C{self._remainingData}. If the payload is complete, checks for the trailing comma and processes the payload. If not, raises an L{IncompleteNetstring} exception. @raise IncompleteNetstring: if the payload received so far contains fewer characters than expected. @raise NetstringParseError: if the payload does not end with a comma. """ self._extractPayload() if self._currentPayloadSize < self._expectedPayloadSize: raise IncompleteNetstring() self._checkForTrailingComma() self._state = self._PARSING_LENGTH self._processPayload() def _extractPayload(self): """ Extracts payload information from C{self._remainingData}. Splits C{self._remainingData} at the end of the netstring. The first part becomes C{self._payload}, the second part is stored in C{self._remainingData}. If the netstring is not yet complete, the whole content of C{self._remainingData} is moved to C{self._payload}. """ if self._payloadComplete(): remainingPayloadSize = (self._expectedPayloadSize - self._currentPayloadSize) self._payload.write(self._remainingData[:remainingPayloadSize]) self._remainingData = self._remainingData[remainingPayloadSize:] self._currentPayloadSize = self._expectedPayloadSize else: self._payload.write(self._remainingData) self._currentPayloadSize += len(self._remainingData) self._remainingData = b"" def _payloadComplete(self): """ Checks if enough data have been received to complete the netstring. @return: C{True} iff the received data contain at least as many characters as specified in the length section of the netstring @rtype: C{bool} """ return (len(self._remainingData) + self._currentPayloadSize >= self._expectedPayloadSize) def _processPayload(self): """ Processes the actual payload with L{stringReceived}. Strips C{self._payload} of the trailing comma and calls L{stringReceived} with the result. """ self.stringReceived(self._payload.getvalue()[:-1]) def _checkForTrailingComma(self): """ Checks if the netstring has a trailing comma at the expected position. @raise NetstringParseError: if the last payload character is anything but a comma. """ if self._payload.getvalue()[-1:] != b",": raise NetstringParseError(self._MISSING_COMMA) def _handleParseError(self): """ Terminates the connection and sets the flag C{self.brokenPeer}. """ self.transport.loseConnection() self.brokenPeer = 1
class Reader(object): def __init__(self, socket, socket_read_size): self._sock = socket self.socket_read_size = socket_read_size self._buffer = BytesIO() self.bytes_written = 0 self.bytes_read = 0 @property def length(self): return self.bytes_written - self.bytes_read def _read_from_socket(self, length=None): socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while True: data = self._sock.recv(socket_read_size) if isinstance(data, bytes) and len(data) == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue break except socket.timeout: raise TimeoutError("Timeout reading from socket") except socket.error: e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, )) def read(self, length): length = length + 2 if length > self.length: self._read_from_socket(length - self.length) self._buffer.seek(self.bytes_read) data = self._buffer.read(length) self.bytes_read += len(data) if self.bytes_read == self.bytes_written: self.purge() return data[:-2] def readline(self): buf = self._buffer buf.seek(self.bytes_read) data = buf.readline() while not data.endswith(SYM_CRLF): self._read_from_socket() buf.seek(self.bytes_read) data = buf.readline() self.bytes_read += len(data) if self.bytes_read == self.bytes_written: self.purge() return data[:-2] def purge(self): self._buffer.seek(0) self._buffer.truncate() self.bytes_written = 0 self.bytes_read = 0 def close(self): try: self.purge() self._buffer.close() except Exception: pass self._buffer = None self._sock = None
camera.start_preview() time.sleep(2) camera.resolution = (640, 480) camera.framerate = 30 while True: try: client = server.accept()[0].makefile('wb') #(client, address) = server.accept() stream = BytesIO() camera.start_recording('/home/pi/FTP/files/vid1.h264', resize=(1920, 1080)) for foo in camera.capture_continuous(stream, 'rgb', splitter_port=2, use_video_port=True): client.write(struct.pack('<L', stream.tell())) #client.sendall(struct.pack('<L', stream.tell())) client.flush() stream.seek(0) client.write(stream.read()) #client.sendall(stream.read()) stream.seek(0) stream.truncate() except socket.error: #server.close() camera.stop_recording() print('Client disconnected')
class CurlHttpEventStream(object): def __init__(self, url, auth, verify): self.url = url self.received_buffer = BytesIO() headers = ['Cache-Control: no-cache', 'Accept: text/event-stream'] self.curl = pycurl.Curl() self.curl.setopt(pycurl.URL, url) self.curl.setopt(pycurl.ENCODING, 'gzip') self.curl.setopt(pycurl.CONNECTTIMEOUT, 10) self.curl.setopt(pycurl.WRITEDATA, self.received_buffer) if auth and type(auth) is DCOSAuth: auth.refresh_auth_header() headers.append('Authorization: %s' % auth.auth_header) elif auth: self.curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) self.curl.setopt(pycurl.USERPWD, '%s:%s' % auth) if verify: self.curl.setopt(pycurl.CA_INFO, verify) self.curl.setopt(pycurl.HTTPHEADER, headers) self.curlmulti = pycurl.CurlMulti() self.curlmulti.add_handle(self.curl) self.status_code = 0 SELECT_TIMEOUT = 10 def _any_data_received(self): return self.received_buffer.tell() != 0 def _get_received_data(self): result = self.received_buffer.getvalue() self.received_buffer.truncate(0) self.received_buffer.seek(0) return result def _check_status_code(self): if self.status_code == 0: self.status_code = self.curl.getinfo(pycurl.HTTP_CODE) if self.status_code != 0 and self.status_code != 200: raise Exception(str(self.status_code) + ' ' + self.url) def _perform_on_curl(self): while True: ret, num_handles = self.curlmulti.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break return num_handles def _iter_chunks(self): while True: remaining = self._perform_on_curl() if self._any_data_received(): self._check_status_code() yield self._get_received_data() if remaining == 0: break self.curlmulti.select(self.SELECT_TIMEOUT) self._check_status_code() self._check_curl_errors() def _check_curl_errors(self): for f in self.curlmulti.info_read()[2]: raise pycurl.error(*f[1:]) def iter_lines(self): chunks = self._iter_chunks() return self._split_lines_from_chunks(chunks) @staticmethod def _split_lines_from_chunks(chunks): #same behaviour as requests' Response.iter_lines(...) pending = None for chunk in chunks: if pending is not None: chunk = pending + chunk lines = chunk.splitlines() if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: pending = lines.pop() else: pending = None for line in lines: yield line if pending is not None: yield pending
def speak(audioString): audioString = audioString.replace('-', ' ').replace('_', ' ') consequitivedots = re.compile(r'\.{2,}') audioString = consequitivedots.sub('.', audioString) sentence = sent(audioString) l = len(sentence) # mixer.init() if tts_ == 'google1': mixer.init(54000, -16, 1, 4096) from io import BytesIO print('[MAIN] - NÓI') print('') print('TTS: Google') for i in range(0, l): text = str(sentence[i]) # text = text[:] print(text) count = count_chars(text) print(count) if count > 0: print('Nội dung: ' + text) #Kiểm tra tồn tại file import os.path from os import path while True: me = path.exists('tmp/google/' + text[:150] + '.mp3') if me == False: tts = gTTS(text, lang='vi') tts.save('tmp/google/' + text[:150] + '.mp3') time_sleep = 1 time_wait = 20 tcount = 0 while (me == False and tcount < time_wait): time.sleep(time_sleep) tts.save('tmp/google/' + text[:150] + '.mp3') tcount += 1 else: if seed == 1: pixels.pixels.speak() audio = MP3('tmp/google/' + text[:150] + '.mp3') t = float(audio.info.length) mixer.music.load('tmp/google/' + text[:150] + '.mp3') mixer.music.play() time.sleep(t) break #FPT elif tts_ == 'fpt': import random, requests for i in range(0, l): text = str(sentence[i]) if len(text.strip()) < 1 or text.strip() == '.': pass else: print('Nội dung: ' + text) import os.path from os import path while True: me = path.exists('tmp/fpt/' + text[:150] + '.mp3') if me == False: api_fpt_key = random.choice(api_fpt) url = 'https://api.fpt.ai/hmi/tts/v5' headers = { 'api_key': api_fpt_key, 'speed': '0', 'prosody': '1', 'voice': voice } payload = text.encode('utf-8') url_return = requests.post( url, data=payload, headers=headers).json()['async'] res_response = requests.get(url_return) res_status = res_response.status_code # print (url_return) time_sleep = 0.5 time_wait = 20 tcount = 0 while (res_status == 404 and tcount < time_wait): time.sleep(time_sleep) res_response = requests.get(url_return) res_status = res_response.status_code tcount += 1 if tcount == time_wait: print('error tts') with open('tmp/fpt/' + text[:150] + '.mp3', 'wb') as f: f.write(res_response.content) f.close() else: mixer.music.load('tmp/fpt/' + text[:150] + '.mp3') mixer.music.play() audio = MP3('tmp/fpt/' + text[:150] + '.mp3') t = float(audio.info.length) print('Time delay :' + str(t)) time.sleep(t) break #TTS VIETTEL elif tts_ == 'viettel': token = gih.get_config('token') import json, requests, os.path from playsound import playsound from pathlib import Path from pygame import mixer url = "https://vtcc.ai/voice/api/tts/v1/rest/syn" for i in range(0, l): text = str(sentence[i]) if len(text.strip()) < 1 or text.strip() == '.': pass else: print('Nội dung: ' + text) import os.path from os import path while True: from pydub import AudioSegment from pydub.playback import play me = path.exists('tmp/vtcc/' + text[:150] + '.wav') if me == False: data = { "text": text, "voice": "hn-quynhanh", "id": "2", "without_filter": False, "speed": 1.0, "tts_return_option": 2 } headers = { 'Content-type': 'application/json', 'token': token } s = requests.Session() dirname = os.path.dirname(os.path.abspath('_file_')) cert_path = (dirname + '/wwwvtccai.crt') response = requests.post(url, data=json.dumps(data), headers=headers, verify=cert_path) print(response.headers) print(response.status_code) res_status = response.status_code time_sleep = 0.5 time_wait = 20 tcount = 0 while (res_status != 200 and tcount < time_wait): time.sleep(time_sleep) response = requests.post(url, data=json.dumps(data), headers=headers, verify=cert_path) res_status = res_response.status_code tcount += 1 if tcount == time_wait: print('error tts') data = response.content with open('tmp/vtcc/' + text[:150] + '.wav', "wb") as f: f.write(data) f.close() else: if seed == 1: pixels.pixels.speak() sound = AudioSegment.from_wav('tmp/vtcc/' + text[:150] + '.wav') play(sound) break # TTS Google1 elif tts_ == 'google': from pygame import mixer mixer.init(54000, -16, 1, 4096) from io import BytesIO print('[MAIN] - NÓI') print('') print('TTS: Google') for i in range(0, l): text = str(sentence[i]) # text = text[:] print(text) count = count_chars(text) print(count) if count > 0: print('Nội dung: ' + text) tts = gTTS(text, lang='vi') mp3_fp = BytesIO() tts.write_to_fp(mp3_fp) t = mp3_fp.truncate() / 5400 t = float(t) mp3_fp.seek(0) mixer.init() if seed == 1: pixels.pixels.speak() mixer.music.load(mp3_fp) mixer.music.play() mp3_fp.flush() mp3_fp.seek(0) time.sleep(t) else: pass
class IdzipWriter(IOStreamWrapperMixin): FILE_EXTENSION = 'dz' enforce_extension = True def __init__(self, output, sync_size=MAX_MEMBER_SIZE, mtime=None): if mtime is None: mtime = time.time() if isinstance(output, basestring): self.output = self._prepare_file_stream(output) self._should_close = True else: # hopefully a file like object if not check_file_like_for_writing(output): raise TypeError( "`output` must be a file-like object supporting " "write, tell, flush, and close!") self.output = output self._should_close = False self.input_buffer = BytesIO() try: name = path.abspath(self.output.name) basename = path.basename(name) self.name = name self.basename = basename.encode(fsencoding) except AttributeError: self.name = "" self.basename = self.name.encode(fsencoding) self.uncompressed_position = 0 self.sync_size = sync_size self.mtime = int(mtime) self.compressobj = None self._reset_compressor() self.version = 1 def _prepare_file_stream(self, path): if self.enforce_extension and not path.endswith(self.FILE_EXTENSION): path = "%s.%s" % (path, self.FILE_EXTENSION) return open(path, 'wb') @property def stream(self): return self.output def _make_compressor(self): return zlib.compressobj(COMPRESSION_LEVEL, zlib.DEFLATED, -zlib.MAX_WBITS) def _reset_compressor(self): self.compressobj = self._make_compressor() def seek(self, offset, whence=SEEK_SET): raise UnsupportedOperation("Cannot seek on a write-only stream") def reset_buffer(self): # For truncate() to actually help us circumvent memory problems, # we need to seek to the beginning of the buffer, otherwise the # next call to write() will just pad the buffer with null bytes # up to the current position returned by tell() self.input_buffer.seek(0) self.input_buffer.truncate(0) def _get_buffer_size(self): curpos = self.input_buffer.tell() self.input_buffer.seek(0, SEEK_END) buffer_len = self.input_buffer.tell() self.input_buffer.seek(curpos) return buffer_len def _write_chunked(self, b, chunk_size=WRITE_BLOCK_SIZE): offset = 0 total = len(b) i = 0 while offset < total: s = b[offset:offset + chunk_size] self.input_buffer.write(s) self.uncompressed_position += len(s) i += 1 offset += chunk_size if i % 2 == 0: self.sync() self.reset_buffer() return total def write(self, b): try: self.input_buffer.write(b) except MemoryError: # Attempting to write to the input buffer may cause # it to optimistically request a contiguous allocation # larger than the memory allocator can currently supply, # raising a MemoryError. This can be recovered from by # syncing the current buffer to disk, resetting the # input buffer, and trying to write again self.sync() self.reset_buffer() try: self.input_buffer.write(b) except MemoryError: # If the data to be written are so large that # they cannot be written to the input buffer in # one shot, try writing it in chunks. If *this* # runs out of memory, we're out of options. return self._write_chunked(b) self.uncompressed_position += len(b) buffer_len = self._get_buffer_size() if buffer_len >= self.sync_size: if self._check_member_size_valid(buffer_len): self.sync() self.reset_buffer() else: self._sync_chunked() return len(b) def sync(self): self.compress_member() self.reset_buffer() return self.output.tell() def _sync_chunked(self, flush=False): # accumulate chunks of the input buffer which are at most # `MAX_MEMBER_SIZE` bytes long valid_members = [] self.input_buffer.seek(0) chunk = self.input_buffer.read(MAX_MEMBER_SIZE - 1) while chunk: valid_members.append(BytesIO(chunk)) chunk = self.input_buffer.read(MAX_MEMBER_SIZE - 1) # if the input buffer was empty, then no valid members were # made so return immediately if not valid_members: return # if there is only one member, chunked syncing should be the same # as normal syncing, leaving the input buffer empty elif len(valid_members) == 1: self.input_buffer = valid_members[0] self.compress_member() self.reset_buffer() else: # otherwise, for all but the last valid member, sync those buffers # to the output stream, and then set the last valid member as the current # input buffer to avoid having a tiny leftover fragment. for chunk in valid_members[:-1]: self.input_buffer = chunk self.compress_member() self.reset_buffer() self.input_buffer = valid_members[-1] if flush: self.compress_member() self.reset_buffer() def tell(self): return self.uncompressed_position def flush(self): self.sync() return self.output.flush() def close(self): if not self.closed: self.sync() self.reset_buffer() if self._should_close: closing = self.output.close() return closing return None def _calculate_number_of_chunks_for_bytes(self, in_size): num_chunks = in_size // CHUNK_LENGTH if in_size % CHUNK_LENGTH != 0: num_chunks += 1 return num_chunks def _calculate_extra_field_length(self, in_size): num_chunks = self._calculate_number_of_chunks_for_bytes(in_size) field_length = 3 * 2 + 2 * num_chunks return field_length def _calculate_header_extra_size(self, in_size): field_length = self._calculate_extra_field_length(in_size) extra_length = 2 * 2 + field_length return extra_length def _check_member_size_valid(self, in_size): extra_length = self._calculate_header_extra_size(in_size) return extra_length <= 0xffff def compress_member(self): """A gzip member contains: 1) The header. 2) The compressed data. """ self.input_buffer.seek(0, SEEK_END) member_size = self.input_buffer.tell() self.input_buffer.seek(0) zlengths_pos = self._prepare_header(member_size) zlengths = self._compress_data(member_size) # Writes the lengths of compressed chunks to the header. end_pos = self.output.tell() self.output.seek(zlengths_pos) for zlen in zlengths: _write16(self.output, zlen) self.output.seek(end_pos) def _prepare_header(self, in_size): """Writes a prepared gzip header to the output. The gzip header is defined in RFC 1952. The gzip header starts with: +---+---+---+---+---+---+---+---+---+---+ |x1f|x8b|x08|FLG| MTIME |XFL|OS | +---+---+---+---+---+---+---+---+---+---+ where: FLG ... flags. FEXTRA|FNAME is used by idzip. MTIME ... the modification time of the original file or 0. XFL ... extra flags about the compression. OS ... operating system used for the compression. The next header sections are: 1) Extra field, if the FEXTRA flag is set. Its format is described in _write_extra_field(). 2) The original file name, if the FNAME flag is set. The file name string is zero-terminated. """ self.output.write(GZIP_DEFLATE_ID) flags = FEXTRA if self.basename: flags |= FNAME self.output.write(bytearray([flags])) # The mtime will be undefined if it does not fit. if self.mtime > 0xffffffff: mtime = 0 else: mtime = self.mtime _write32(self.output, mtime) deflate_flags = b"\0" if COMPRESSION_LEVEL == zlib.Z_BEST_COMPRESSION: deflate_flags = b"\x02" # slowest compression algorithm self.output.write(deflate_flags) self.output.write(bytearray([OS_CODE_UNIX])) zlengths_pos = self._write_extra_field(in_size) if self.basename: self.output.write(self.basename + b'\0') # original basename return zlengths_pos def _write_extra_field(self, in_size): """Writes the dictzip extra field. It will be initiated with zeros on the place of the lengths of compressed chunks. The gzip extra field is present when the FEXTRA flag is set. RFC 1952 defines the used bytes: +---+---+================================+ | XLEN | XLEN bytes of "extra field" ...| +---+---+================================+ Idzip adds only one subfield: +---+---+---+---+===============================+ |'R'|'A'| LEN | LEN bytes of subfield data ...| +---+---+---+---+===============================+ The subfield ID "RA" stands for Random Access. That subfield ID signalizes the dictzip gzip extension. The dictzip stores the length of uncompressed chunks and the lengths of compressed chunks to the gzip header: +---+---+---+---+---+---+==============================================+ | VER=1 | CHLEN | CHCNT | CHCNT 2-byte lengths of compressed chunks ...| +---+---+---+---+---+---+==============================================+ Two bytes are used to store a length of a compressed chunk. So the length of a compressed chunk has to be max 0xfffff. That puts a restriction on the CHLEN -- the length of uncompressed chunks. Dictzip uses CHLEN=58315. Only a fixed number of chunk lengths will fit to the gzip header. That limits the max file size of a dictzip file. Idzip does not have that limitation. It starts a new gzip member if needed. The new member would be also a valid dictzip file. """ # num_chunks = in_size // CHUNK_LENGTH # if in_size % CHUNK_LENGTH != 0: # num_chunks += 1 num_chunks = self._calculate_number_of_chunks_for_bytes(in_size) # field_length = 3 * 2 + 2 * num_chunks field_length = self._calculate_extra_field_length(in_size) # extra_length = 2 * 2 + field_length extra_length = self._calculate_header_extra_size(in_size) assert extra_length <= 0xffff _write16(self.output, extra_length) # XLEN # Dictzip extra field (Random Access) self.output.write(b"RA") _write16(self.output, field_length) _write16(self.output, self.version) # version _write16(self.output, CHUNK_LENGTH) _write16(self.output, num_chunks) zlengths_pos = self.output.tell() self.output.write(b"\0\0" * num_chunks) return zlengths_pos def _compress_data(self, in_size): """Compresses the given number of input bytes to the output. The output consists of: 1) The compressed data. 2) 4 bytes of CRC. 3) 4 bytes of file size. """ assert in_size <= 0xffffffff zlengths = [] crcval = zlib.crc32(b"") need = in_size while need > 0: read_size = min(need, CHUNK_LENGTH) chunk = self.input_buffer.read(read_size) if len(chunk) != read_size: raise IOError("Need %s bytes, got %s" % (read_size, len(chunk))) need -= len(chunk) crcval = zlib.crc32(chunk, crcval) zlen = self._compress_chunk(chunk) zlengths.append(zlen) # An empty block with BFINAL=1 flag ends the zlib data stream. self.output.write(self.compressobj.flush(zlib.Z_FINISH)) self._reset_compressor() _write32(self.output, crcval) _write32(self.output, in_size) return zlengths def _compress_chunk(self, chunk): data = self.compressobj.compress(chunk) zlen = len(data) self.output.write(data) data = self.compressobj.flush(zlib.Z_FULL_FLUSH) zlen += len(data) self.output.write(data) return zlen
def test_genZshFunction(self, cmdName, optionsFQPN): """ Generate completion functions for given twisted command - no errors should be raised @type cmdName: C{str} @param cmdName: The name of the command-line utility e.g. 'twistd' @type optionsFQPN: C{str} @param optionsFQPN: The Fully Qualified Python Name of the C{Options} class to be tested. """ outputFile = BytesIO() self.patch(usage.Options, "_shellCompFile", outputFile) # some scripts won't import or instantiate because of missing # dependencies (pyOpenSSL, etc) so we have to skip them. try: o = reflect.namedAny(optionsFQPN)() except Exception as e: raise unittest.SkipTest( "Couldn't import or instantiate " "Options class: %s" % (e,) ) try: o.parseOptions(["", "--_shell-completion", "zsh:2"]) except ImportError as e: # this can happen for commands which don't have all # the necessary dependencies installed. skip test. # skip raise unittest.SkipTest("ImportError calling parseOptions(): %s", (e,)) except SystemExit: pass # expected else: self.fail("SystemExit not raised") outputFile.seek(0) # test that we got some output self.assertEqual(1, len(outputFile.read(1))) outputFile.seek(0) outputFile.truncate() # now, if it has sub commands, we have to test those too if hasattr(o, "subCommands"): for (cmd, short, parser, doc) in o.subCommands: try: o.parseOptions([cmd, "", "--_shell-completion", "zsh:3"]) except ImportError as e: # this can happen for commands which don't have all # the necessary dependencies installed. skip test. raise unittest.SkipTest( "ImportError calling parseOptions() " "on subcommand: %s", (e,) ) except SystemExit: pass # expected else: self.fail("SystemExit not raised") outputFile.seek(0) # test that we got some output self.assertEqual(1, len(outputFile.read(1))) outputFile.seek(0) outputFile.truncate() # flushed because we don't want DeprecationWarnings to be printed when # running these test cases. self.flushWarnings()
class _ResponseReader(protocol.Protocol): def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 self._certificate = None self._ip_address = None self._crawler = crawler def _finish_response(self, flags=None, failure=None): self._finished.callback({ "txresponse": self._txresponse, "body": self._bodybuf.getvalue(), "flags": flags, "certificate": self._certificate, "ip_address": self._ip_address, "failure": failure, }) def connectionMade(self): if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate( self.transport._producer.getPeerCertificate()) if self._ip_address is None: self._ip_address = ipaddress.ip_address( self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. if self._finished.called: return self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) bytes_received_result = self._crawler.signals.send_catch_log( signal=signals.bytes_received, data=bodyBytes, request=self._request, spider=self._crawler.spider, ) for handler, result in bytes_received_result: if isinstance(result, Failure) and isinstance( result.value, StopDownload): logger.debug( "Download stopped for %(request)s from signal handler %(handler)s", { "request": self._request, "handler": handler.__qualname__ }) self.transport.stopProducing() self.transport._producer.loseConnection() failure = result if result.value.fail else None self._finish_response(flags=["download_stopped"], failure=failure) if self._maxsize and self._bytes_received > self._maxsize: logger.warning( "Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", { 'bytes': self._bytes_received, 'maxsize': self._maxsize, 'request': self._request }) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: self._reached_warnsize = True logger.warning( "Received more bytes than download " "warn size (%(warnsize)s) in request %(request)s.", { 'warnsize': self._warnsize, 'request': self._request }) def connectionLost(self, reason): if self._finished.called: return if reason.check(ResponseDone): self._finish_response() return if reason.check(PotentialDataLoss): self._finish_response(flags=["partial"]) return if reason.check(ResponseFailed) and any( r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: self._finish_response(flags=["dataloss"]) return elif not self._fail_on_dataloss_warned: logger.warning( "Got data loss in %s. If you want to process broken " "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" " -- This message won't be shown in further requests", self._txresponse.request.absoluteURI.decode()) self._fail_on_dataloss_warned = True self._finished.errback(reason)
class THttpClient(TTransportBase): """Http implementation of TTransport base.""" def __init__(self, uri_or_host, port=None, path=None, cafile=None, cert_file=None, key_file=None, ssl_context=None): if port is not None: warnings.warn( "Please use the THttpClient('http{s}://host:port/path') constructor", DeprecationWarning, stacklevel=2) self.host = uri_or_host self.port = port assert path self.path = path self.scheme = 'http' else: parsed = urllib.parse.urlparse(uri_or_host) self.scheme = parsed.scheme assert self.scheme in ('http', 'https') if self.scheme == 'http': self.port = parsed.port or http_client.HTTP_PORT elif self.scheme == 'https': self.port = parsed.port or http_client.HTTPS_PORT self.certfile = cert_file self.keyfile = key_file self.context = ssl.create_default_context(cafile=cafile) if (cafile and not ssl_context) else ssl_context self.host = parsed.hostname self.path = parsed.path if parsed.query: self.path += '?%s' % parsed.query proxy = None self.realhost = self.realport = self.proxy_auth = None self.__wbuf = BytesIO() self.__rbuf = BytesIO() self.__http = None self.__http_response = None self.__timeout = None self.__custom_headers = None self.__time = time.time() def using_proxy(self): return self.realhost is not None def open(self): if self.scheme == 'http': self.__http = http_client.HTTPConnection(self.host, self.port, timeout=self.__timeout) if self.scheme == 'https': self.__http = http_client.HTTPSConnection(self.host, self.port, key_file=self.keyfile, cert_file=self.certfile, timeout=self.__timeout, context=self.context) def close(self): self.__http.close() self.__http = None self.__http_response = None def isOpen(self): return self.__http is not None def setTimeout(self, ms): if ms is None: self.__timeout = None else: self.__timeout = ms / 1000.0 def setCustomHeaders(self, headers): self.__custom_headers = headers def read(self, sz): return self.__rbuf.read(sz) def write(self, buf): self.__wbuf.write(buf) def flush(self): if not self.isOpen(): self.open(); self.__time = time.time() elif time.time() - self.__time > 90: self.close(); self.open(); self.__time = time.time() data = self.__wbuf.getvalue() self.__wbuf.truncate(0) self.__wbuf.seek(0) self.__http.putrequest('POST', self.path) self.__http.putheader('Content-Type', 'application/x-thrift') self.__http.putheader('Content-Length', str(len(data))) if self.__custom_headers: for key, val in six.iteritems(self.__custom_headers): self.__http.putheader(key, val) self.__http.endheaders() self.__http.send(data) self.__rbuf = BytesIO(self.__http.getresponse().read())
class Headers: header_size = 112 chunk_size = 10**16 max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' target_timespan = 150 checkpoints = HASHES first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot timestamp_average_offset = 160.6855883050695 # calculated at 733447 validate_difficulty: bool = True def __init__(self, path) -> None: self.io = None self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None self.known_missing_checkpointed_chunks = set() self.check_chunk_lock = asyncio.Lock() async def open(self): self.io = BytesIO() if self.path != ':memory:': def _readit(): if os.path.exists(self.path): with open(self.path, 'r+b') as header_file: self.io.seek(0) self.io.write(header_file.read()) await asyncio.get_event_loop().run_in_executor(None, _readit) bytes_size = self.io.seek(0, os.SEEK_END) self._size = bytes_size // self.header_size max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 if bytes_size % self.header_size: log.warning("Reader file size doesnt match header size. Repairing, might take a while.") await self.repair() else: # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok) await self.repair(start_height=max_checkpointed_height) await self.ensure_checkpointed_size() await self.get_all_missing_headers() async def close(self): if self.io is not None: def _close(): flags = 'r+b' if os.path.exists(self.path) else 'w+b' with open(self.path, flags) as header_file: header_file.write(self.io.getbuffer()) await asyncio.get_event_loop().run_in_executor(None, _close) self.io.close() self.io = None @staticmethod def serialize(header): return b''.join([ struct.pack('<I', header['version']), unhexlify(header['prev_block_hash'])[::-1], unhexlify(header['merkle_root'])[::-1], unhexlify(header['claim_trie_root'])[::-1], struct.pack('<III', header['timestamp'], header['bits'], header['nonce']) ]) @staticmethod def deserialize(height, header): version, = struct.unpack('<I', header[:4]) timestamp, bits, nonce = struct.unpack('<III', header[100:112]) return { 'version': version, 'prev_block_hash': hexlify(header[4:36][::-1]), 'merkle_root': hexlify(header[36:68][::-1]), 'claim_trie_root': hexlify(header[68:100][::-1]), 'timestamp': timestamp, 'bits': bits, 'nonce': nonce, 'block_height': height, } def get_next_chunk_target(self, chunk: int) -> ArithUint256: return ArithUint256(self.max_target) def get_next_block_target(self, max_target: ArithUint256, previous: Optional[dict], current: Optional[dict]) -> ArithUint256: # https://github.com/lbryio/lbrycrd/blob/master/src/lbry.cpp if previous is None and current is None: return max_target if previous is None: previous = current actual_timespan = current['timestamp'] - previous['timestamp'] modulated_timespan = self.target_timespan + int((actual_timespan - self.target_timespan) / 8) minimum_timespan = self.target_timespan - int(self.target_timespan / 8) # 150 - 18 = 132 maximum_timespan = self.target_timespan + int(self.target_timespan / 2) # 150 + 75 = 225 clamped_timespan = max(minimum_timespan, min(modulated_timespan, maximum_timespan)) target = ArithUint256.from_compact(current['bits']) new_target = min(max_target, (target * clamped_timespan) / self.target_timespan) return new_target def __len__(self) -> int: return self._size def __bool__(self): return True async def get(self, height) -> dict: if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") try: return self.deserialize(height, await self.get_raw_header(height)) except struct.error: raise IndexError(f"failed to get {height}, at {len(self)}") def estimated_timestamp(self, height, try_real_headers=True): if height <= 0: return if try_real_headers and self.has_header(height): offset = height * self.header_size return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0] return int(self.first_block_timestamp + (height * self.timestamp_average_offset)) def estimated_julian_day(self, height): return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False))) async def get_raw_header(self, height) -> bytes: if self.chunk_getter: await self.ensure_chunk_at(height) if not 0 <= height <= self.height: raise IndexError(f"{height} is out of bounds, current height: {self.height}") return self._read(height) def _read(self, height, count=1): offset = height * self.header_size return bytes(self.io.getbuffer()[offset: offset + self.header_size * count]) def chunk_hash(self, start, count): return self.hash_header(self._read(start, count)).decode() async def ensure_checkpointed_size(self): max_checkpointed_height = max(self.checkpoints.keys() or [-1]) if self.height < max_checkpointed_height: self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000)) async def ensure_chunk_at(self, height): async with self.check_chunk_lock: if self.has_header(height): log.debug("has header %s", height) return return await self.fetch_chunk(height) async def fetch_chunk(self, height): log.info("on-demand fetching height %s", height) start = (height // 1000) * 1000 headers = await self.chunk_getter(start) # pylint: disable=not-callable chunk = ( zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) ) chunk_hash = self.hash_header(chunk).decode() if self.checkpoints.get(start) == chunk_hash: self._write(start, chunk) if start in self.known_missing_checkpointed_chunks: self.known_missing_checkpointed_chunks.remove(start) return elif start not in self.checkpoints: return # todo: fixme raise Exception( f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead." ) def has_header(self, height): normalized_height = (height // 1000) * 1000 if normalized_height in self.checkpoints: return normalized_height not in self.known_missing_checkpointed_chunks empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' return self.chunk_hash(height, 1) not in (empty, all_zeroes) async def get_all_missing_headers(self): # Heavy operation done in one optimized shot for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): if chunk_height in self.known_missing_checkpointed_chunks: continue if self.chunk_hash(chunk_height, 1000) != expected_hash: self.known_missing_checkpointed_chunks.add(chunk_height) return self.known_missing_checkpointed_chunks @property def height(self) -> int: return len(self)-1 @property def bytes_size(self): return len(self) * self.header_size async def hash(self, height=None) -> bytes: return self.hash_header( await self.get_raw_header(height if height is not None else self.height) ) @staticmethod def hash_header(header: bytes) -> bytes: if header is None: return b'0' * 64 return hexlify(double_sha256(header)[::-1]) async def connect(self, start: int, headers: bytes) -> int: added = 0 bail = False for height, chunk in self._iterate_chunks(start, headers): try: # validate_chunk() is CPU bound and reads previous chunks from file system await self.validate_chunk(height, chunk) except InvalidHeader as e: bail = True chunk = chunk[:(height-e.height)*self.header_size] if chunk: added += self._write(height, chunk) if bail: break return added def _write(self, height, verified_chunk): self.io.seek(height * self.header_size, os.SEEK_SET) written = self.io.write(verified_chunk) // self.header_size # self.io.truncate() # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() self._size = max(self._size or 0, self.io.tell() // self.header_size) return written async def validate_chunk(self, height, chunk): previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: raw = await self.get_raw_header(height-1) previous_header = self.deserialize(height-1, raw) previous_hash = self.hash_header(raw) if height > 1: previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) for current_hash, current_header in self._iterate_headers(height, chunk): block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header) self.validate_header(height, current_hash, current_header, previous_hash, block_target) previous_previous_header = previous_header previous_header = current_header previous_hash = current_hash def validate_header(self, height: int, current_hash: bytes, header: dict, previous_hash: bytes, target: ArithUint256): if previous_hash is None: if self.genesis_hash is not None and self.genesis_hash != current_hash: raise InvalidHeader( height, f"genesis header doesn't match: {current_hash.decode()} " f"vs expected {self.genesis_hash.decode()}") return if header['prev_block_hash'] != previous_hash: raise InvalidHeader( height, "previous hash mismatch: {} vs expected {}".format( header['prev_block_hash'].decode(), previous_hash.decode()) ) if self.validate_difficulty: if header['bits'] != target.compact: raise InvalidHeader( height, "bits mismatch: {} vs expected {}".format( header['bits'], target.compact) ) proof_of_work = self.get_proof_of_work(current_hash) if proof_of_work > target: raise InvalidHeader( height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" ) async def repair(self, start_height=0): previous_header_hash = fail = None batch_size = 36 for height in range(start_height, self.height, batch_size): headers = self._read(height, batch_size) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] if previous_header_hash: if header['prev_block_hash'] != previous_header_hash: fail = True elif height == 0: if header_hash != self.genesis_hash: fail = True else: # for sanity and clarity, since it is the only way we can end up here assert start_height > 0 and height == start_height if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) self.io.truncate() self.io.flush() self._size = self.io.seek(0, os.SEEK_END) // self.header_size return previous_header_hash = header_hash @classmethod def get_proof_of_work(cls, header_hash: bytes): return ArithUint256(int(b'0x' + cls.header_hash_to_pow_hash(header_hash), 16)) def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]: assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}" start = 0 end = (self.chunk_size - height % self.chunk_size) * self.header_size while start < end: yield height + (start // self.header_size), headers[start:end] start = end end = min(len(headers), end + self.chunk_size * self.header_size) def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]: assert len(headers) % self.header_size == 0, len(headers) for idx in range(len(headers) // self.header_size): start, end = idx * self.header_size, (idx + 1) * self.header_size header = headers[start:end] yield self.hash_header(header), self.deserialize(height+idx, header) @staticmethod def header_hash_to_pow_hash(header_hash: bytes): header_hash_bytes = unhexlify(header_hash)[::-1] h = sha512(header_hash_bytes) pow_hash = double_sha256( ripemd160(h[:len(h) // 2]) + ripemd160(h[len(h) // 2:]) ) return hexlify(pow_hash[::-1])
def localize(fp, **kw): """ Generate em:localized """ kw = kw def sort(f): if "en-US" in f["locale"]: return 0, f["locale"] return 1, f["locale"] def get_locale_strings(): locales = list() for f in sorted(files("chrome/locale/*/description.properties")): locale = dict(locale=(f.split("/", 3)[2], )) with open(f, "r", encoding="utf-8") as lp: for l in lp: l = l.strip() if not l or l.startswith("#"): continue k, v = l.split("=", 1) k = k.split(".") k = k[-2] if len(k[-1]) < 3 else k[-1] if not k or not v: continue if k not in locale: locale[k] = list() locale[k] += v, locales += locale, return locales locales = get_locale_strings() with Reset(fp): rdf = XML(fp.read()) # kill old localized for e in rdf.getElementsByTagNameNS(NS_EM, "localized"): e.parentNode.removeChild(e) def mapkey(k): v = list() for e in rdf.getElementsByTagNameNS(NS_EM, k): v += e.firstChild.data, return k, v keys = ("locale", "name", "description", "creator", "homepageURL", "developer", "translator", "contributor") baseprops = dict(mapkey(k) for k in keys) def add_props(): parent = rdf.getElementsByTagNameNS(NS_EM, "id")[0].parentNode for props in sorted(locales, key=sort): node = rdf.createElementNS(NS_EM, "em:localized") desc = rdf.createElementNS(NS_RDF, "Description") for k in keys: vals = props.get(k, baseprops.get(k, list())) for v in vals: n = rdf.createElementNS(NS_EM, "em:" + k) n.appendChild(rdf.createTextNode(v)) desc.appendChild(n) parent.appendChild(rdf.createTextNode("\n\t\t")) node.appendChild(desc) parent.appendChild(node) parent.appendChild(rdf.createTextNode("\n\t")) add_props() io = BytesIO() with Reset(io): io.write(rdf.toxml(encoding="utf-8")) io.truncate() rdf.unlink() return io
class TCPROSTransport(Transport): """ Generic implementation of TCPROS exchange routines for both topics and services """ transport_type = 'TCPROS' def __init__(self, protocol, name, header=None): """ ctor @param name str: identifier @param protocol TCPROSTransportProtocol protocol implementation @param header dict: (optional) handshake header if transport handshake header was already read off of transport. @raise TransportInitError if transport cannot be initialized according to arguments """ super(TCPROSTransport, self).__init__(protocol.direction, name=name) if not name: raise TransportInitError( "Unable to initialize transport: name is not set") self.protocol = protocol self.socket = None self.endpoint_id = 'unknown' self.dest_address = None # for reconnection if python3 == 0: # Python 2.x self.read_buff = StringIO() self.write_buff = StringIO() else: # Python 3.x self.read_buff = BytesIO() self.write_buff = BytesIO() #self.write_buff = StringIO() self.header = header # #1852 have to hold onto latched messages on subscriber side self.is_latched = False self.latch = None # save the fileno separately so we can garbage collect the # socket but still unregister will poll objects self._fileno = None # these fields are actually set by the remote # publisher/service. they are set for tools that connect # without knowing the actual field name self.md5sum = None self.type = None def fileno(self): """ Get descriptor for select """ return self._fileno def set_socket(self, sock, endpoint_id): """ Set the socket for this transport @param sock: socket @type sock: socket.socket @param endpoint_id: identifier for connection endpoint @type endpoint_id: str @raise TransportInitError: if socket has already been set """ if self.socket is not None: raise TransportInitError("socket already initialized") self.socket = sock self.endpoint_id = endpoint_id self._fileno = sock.fileno() def connect(self, dest_addr, dest_port, endpoint_id, timeout=None): """ Establish TCP connection to the specified address/port. connect() always calls L{write_header()} and L{read_header()} after the connection is made @param dest_addr: destination IP address @type dest_addr: str @param dest_port: destination port @type dest_port: int @param endpoint_id: string identifier for connection (for statistics) @type endpoint_id: str @param timeout: (optional keyword) timeout in seconds @type timeout: float @raise TransportInitError: if unable to create connection """ try: self.endpoint_id = endpoint_id self.dest_address = (dest_addr, dest_port) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if _is_use_tcp_keepalive(): # OSX (among others) does not define these options if hasattr(socket, 'TCP_KEEPCNT') and \ hasattr(socket, 'TCP_KEEPIDLE') and \ hasattr(socket, 'TCP_KEEPINTVL'): # turn on KEEPALIVE s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # - # keepalive failures before actual connection failure s.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 9) # - timeout before starting KEEPALIVE process s.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 60) # - interval to send KEEPALIVE after IDLE timeout s.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 10) if timeout is not None: s.settimeout(timeout) self.socket = s self.socket.connect((dest_addr, dest_port)) self.write_header() self.read_header() except TransportInitError as tie: rospyerr( "Unable to initiate TCP/IP socket to %s:%s (%s): %s" % (dest_addr, dest_port, endpoint_id, traceback.format_exc())) raise except Exception as e: #logerr("Unknown error initiating TCP/IP socket to %s:%s (%s): %s"%(dest_addr, dest_port, endpoint_id, str(e))) rospywarn( "Unknown error initiating TCP/IP socket to %s:%s (%s): %s" % (dest_addr, dest_port, endpoint_id, traceback.format_exc())) # FATAL: no reconnection as error is unknown self.done = True if self.socket: try: self.socket.shutdown(socket.SHUT_RDWR) except: pass finally: self.socket.close() self.socket = None raise TransportInitError(str(e)) #re-raise i/o error def _validate_header(self, header): """ Validate header and initialize fields accordingly @param header: header fields from publisher @type header: dict @raise TransportInitError: if header fails to validate """ self.header = header if 'error' in header: raise TransportInitError("remote error reported: %s" % header['error']) for required in ['md5sum', 'type']: if not required in header: raise TransportInitError("header missing required field [%s]" % required) self.md5sum = header['md5sum'] self.type = header['type'] if header.get('latching', '0') == '1': self.is_latched = True def write_header(self): """Writes the TCPROS header to the active connection.""" # socket may still be getting spun up, so wait for it to be writable sock = self.socket protocol = self.protocol # race condition on close, better fix is to pass these in, # functional style, but right now trying to cause minimal # perturbance to codebase. if sock is None or protocol is None: return fileno = sock.fileno() ready = None while not ready: _, ready, _ = select.select([], [fileno], []) logger.debug("[%s]: writing header", self.name) sock.setblocking(1) self.stat_bytes += write_ros_handshake_header( sock, protocol.get_header_fields()) def read_header(self): """ Read TCPROS header from active socket @raise TransportInitError if header fails to validate """ sock = self.socket if sock is None: return sock.setblocking(1) self._validate_header( read_ros_handshake_header(sock, self.read_buff, self.protocol.buff_size)) def send_message(self, msg, seq): """ Convenience routine for services to send a message across a particular connection. NOTE: write_data is much more efficient if same message is being sent to multiple connections. Not threadsafe. @param msg: message to send @type msg: Msg @param seq: sequence number for message @type seq: int @raise TransportException: if error occurred sending message """ # this will call write_data(), so no need to keep track of stats serialize_message(self.write_buff, seq, msg) self.write_data(self.write_buff.getvalue()) self.write_buff.truncate(0) def write_data(self, data): """ Write raw data to transport @raise TransportInitialiationError: could not be initialized @raise TransportTerminated: no longer open for publishing """ if not self.socket: raise TransportInitError( "TCPROS transport was not successfully initialized") if self.done: raise TransportTerminated("connection closed") try: #TODO: get rid of sendalls and replace with async-style publishing self.socket.sendall(data) self.stat_bytes += len(data) self.stat_num_msg += 1 except IOError as ioe: #for now, just document common errno's in code (errno, msg) = ioe.args if errno == 32: #broken pipe logdebug("ERROR: Broken Pipe") self.close() raise TransportTerminated(str(errno) + msg) raise #re-raise except socket.error as se: #for now, just document common errno's in code (errno, msg) = se.args if errno == 32: #broken pipe logdebug("[%s]: Closing connection [%s] due to broken pipe", self.name, self.endpoint_id) self.close() raise TransportTerminated(msg) elif errno == 104: #connection reset by peer logdebug("[%s]: Peer [%s] has closed connection", self.name, self.endpoint_id) self.close() raise TransportTerminated(msg) else: rospydebug("unknown socket error writing data: %s", traceback.format_exc()) logdebug( "[%s]: closing connection [%s] due to unknown socket error: %s", self.name, self.endpoint_id, msg) self.close() raise TransportTerminated(str(errno) + ' ' + msg) return True def receive_once(self): """ block until messages are read off of socket @return: list of newly received messages @rtype: [Msg] @raise TransportException: if unable to receive message due to error """ sock = self.socket if sock is None: raise TransportException("connection not initialized") b = self.read_buff msg_queue = [] p = self.protocol try: sock.setblocking(1) while not msg_queue and not self.done and not is_shutdown(): if b.tell() >= 4: p.read_messages(b, msg_queue, sock) if not msg_queue: recv_buff(sock, b, p.buff_size) self.stat_num_msg += len(msg_queue) #STATS # set the _connection_header field for m in msg_queue: m._connection_header = self.header # #1852: keep track of last latched message if self.is_latched and msg_queue: self.latch = msg_queue[-1] return msg_queue except DeserializationError as e: rospyerr(traceback.format_exc()) raise TransportException( "receive_once[%s]: DeserializationError %s" % (self.name, str(e))) except TransportTerminated as e: raise #reraise except ServiceException as e: raise except Exception as e: rospyerr(traceback.format_exc()) raise TransportException("receive_once[%s]: unexpected error %s" % (self.name, str(e))) return retval def _reconnect(self): # This reconnection logic is very hacky right now. I need to # rewrite the I/O core so that this is handled more centrally. if self.dest_address is None: raise ROSInitException( "internal error with reconnection state: address not stored") interval = 0.5 # seconds while self.socket is None and not self.done and not rospy.is_shutdown( ): try: # set a timeout so that we can continue polling for # exit. 30. is a bit high, but I'm concerned about # embedded platforms. To do this properly, we'd have # to move to non-blocking routines. self.connect(self.dest_address[0], self.dest_address[1], self.endpoint_id, timeout=30.) except TransportInitError: self.socket = None if self.socket is None: # exponential backoff interval = interval * 2 time.sleep(interval) def receive_loop(self, msgs_callback): """ Receive messages until shutdown @param msgs_callback: callback to invoke for new messages received @type msgs_callback: fn([msg]) """ # - use assert here as this would be an internal error, aka bug logger.debug("receive_loop for [%s]", self.name) try: while not self.done and not is_shutdown(): try: if self.socket is not None: msgs = self.receive_once() if not self.done and not is_shutdown(): msgs_callback(msgs) else: self._reconnect() except TransportException as e: # set socket to None so we reconnect try: if self.socket is not None: try: self.socket.shutdown() except: pass finally: self.socket.close() except: pass self.socket = None except DeserializationError as e: #TODO: how should we handle reconnect in this case? logerr( "[%s] error deserializing incoming request: %s" % self.name, str(e)) rospyerr( "[%s] error deserializing incoming request: %s" % self.name, traceback.format_exc()) except: # in many cases this will be a normal hangup, but log internally try: #1467 sometimes we get exceptions due to #interpreter shutdown, so blanket ignore those if #the reporting fails rospydebug( "exception in receive loop for [%s], may be normal. Exception is %s", self.name, traceback.format_exc()) except: pass rospydebug("receive_loop[%s]: done condition met, exited loop" % self.name) finally: if not self.done: self.close() def close(self): """close i/o and release resources""" if not self.done: try: if self.socket is not None: try: self.socket.shutdown(socket.SHUT_RDWR) except: pass finally: self.socket.close() finally: self.socket = self.read_buff = self.write_buff = self.protocol = None super(TCPROSTransport, self).close()
class THeaderTransport(TTransportBase, CReadableTransport): """Transport that sends headers. Also understands framed/unframed/HTTP transports and will do the right thing""" __max_frame_size = MAX_FRAME_SIZE # Defaults to current user, but there is also a setter below. __identity = None IDENTITY_HEADER = "identity" ID_VERSION_HEADER = "id_version" ID_VERSION = "1" def __init__(self, trans, client_types=None, client_type=None): self.__trans = trans self.__rbuf = StringIO() self.__rbuf_frame = False self.__wbuf = StringIO() self.seq_id = 0 self.__flags = 0 self.__read_transforms = [] self.__write_transforms = [] self.__supported_client_types = set(client_types or (CLIENT_TYPE.HEADER, )) self.__proto_id = T_COMPACT_PROTOCOL # default to compact like c++ self.__client_type = client_type or CLIENT_TYPE.HEADER self.__read_headers = {} self.__read_persistent_headers = {} self.__write_headers = {} self.__write_persistent_headers = {} self.__supported_client_types.add(self.__client_type) # If we support unframed binary / framed binary also support compact if CLIENT_TYPE.UNFRAMED_DEPRECATED in self.__supported_client_types: self.__supported_client_types.add( CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED) if CLIENT_TYPE.FRAMED_DEPRECATED in self.__supported_client_types: self.__supported_client_types.add(CLIENT_TYPE.FRAMED_COMPACT) def set_header_flag(self, flag): self.__flags |= flag def clear_header_flag(self, flag): self.__flags &= ~flag def header_flags(self): return self.__flags def set_max_frame_size(self, size): if size > MAX_BIG_FRAME_SIZE: raise TTransportException( TTransportException.INVALID_FRAME_SIZE, "Cannot set max frame size > %s" % MAX_BIG_FRAME_SIZE) if size > MAX_FRAME_SIZE and self.__client_type != CLIENT_TYPE.HEADER: raise TTransportException( TTransportException.INVALID_FRAME_SIZE, "Cannot set max frame size > %s for clients other than HEADER" % MAX_FRAME_SIZE) self.__max_frame_size = size def get_peer_identity(self): if self.IDENTITY_HEADER in self.__read_headers: if self.__read_headers[self.ID_VERSION_HEADER] == self.ID_VERSION: return self.__read_headers[self.IDENTITY_HEADER] return None def set_identity(self, identity): self.__identity = identity def get_protocol_id(self): return self.__proto_id def set_protocol_id(self, proto_id): self.__proto_id = proto_id def set_header(self, str_key, str_value): self.__write_headers[str_key] = str_value def get_write_headers(self): return self.__write_headers def get_headers(self): return self.__read_headers def clear_headers(self): self.__write_headers.clear() def set_persistent_header(self, str_key, str_value): self.__write_persistent_headers[str_key] = str_value def get_write_persistent_headers(self): return self.__write_persistent_headers def clear_persistent_headers(self): self.__write_persistent_headers.clear() def add_transform(self, trans_id): self.__write_transforms.append(trans_id) def _reset_protocol(self): # HTTP calls that are one way need to flush here. if self.__client_type == CLIENT_TYPE.HTTP_SERVER: self.flush() # set to anything except unframed self.__client_type = CLIENT_TYPE.UNKNOWN # Read header bytes to check which protocol to decode self.readFrame(0) def getTransport(self): return self.__trans def isOpen(self): return self.getTransport().isOpen() def open(self): return self.getTransport().open() def close(self): return self.getTransport().close() def read(self, sz): ret = self.__rbuf.read(sz) if len(ret) == sz: return ret if self.__client_type in (CLIENT_TYPE.UNFRAMED_DEPRECATED, CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED): return ret + self.getTransport().readAll(sz - len(ret)) self.readFrame(sz - len(ret)) return ret + self.__rbuf.read(sz - len(ret)) readAll = read # TTransportBase.readAll does a needless copy here. def readFrame(self, req_sz): self.__rbuf_frame = True word1 = self.getTransport().readAll(4) sz = unpack('!I', word1)[0] proto_id = word1[0] if PY3 else ord(word1[0]) if proto_id == TBinaryProtocol.PROTOCOL_ID: # unframed self.__client_type = CLIENT_TYPE.UNFRAMED_DEPRECATED self.__proto_id = T_BINARY_PROTOCOL if req_sz <= 4: # check for reads < 0. self.__rbuf = StringIO(word1) else: self.__rbuf = StringIO(word1 + self.getTransport().read(req_sz - 4)) elif proto_id == TCompactProtocol.PROTOCOL_ID: self.__client_type = CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED self.__proto_id = T_COMPACT_PROTOCOL if req_sz <= 4: # check for reads < 0. self.__rbuf = StringIO(word1) else: self.__rbuf = StringIO(word1 + self.getTransport().read(req_sz - 4)) elif sz == HTTP_SERVER_MAGIC: self.__client_type = CLIENT_TYPE.HTTP_SERVER mf = self.getTransport().handle.makefile('rb', -1) self.handler = RequestHandler(mf, 'client_address:port', '') self.header = self.handler.wfile self.__rbuf = StringIO(self.handler.data) else: if sz == BIG_FRAME_MAGIC: sz = unpack('!Q', self.getTransport().readAll(8))[0] # could be header format or framed. Check next two bytes. magic = self.getTransport().readAll(2) proto_id = magic[0] if PY3 else ord(magic[0]) if proto_id == TCompactProtocol.PROTOCOL_ID: self.__client_type = CLIENT_TYPE.FRAMED_COMPACT self.__proto_id = T_COMPACT_PROTOCOL _frame_size_check(sz, self.__max_frame_size, header=False) self.__rbuf = StringIO(magic + self.getTransport().readAll(sz - 2)) elif proto_id == TBinaryProtocol.PROTOCOL_ID: self.__client_type = CLIENT_TYPE.FRAMED_DEPRECATED self.__proto_id = T_BINARY_PROTOCOL _frame_size_check(sz, self.__max_frame_size, header=False) self.__rbuf = StringIO(magic + self.getTransport().readAll(sz - 2)) elif magic == PACKED_HEADER_MAGIC: self.__client_type = CLIENT_TYPE.HEADER _frame_size_check(sz, self.__max_frame_size) # flags(2), seq_id(4), header_size(2) n_header_meta = self.getTransport().readAll(8) self.__flags, self.seq_id, header_size = unpack( '!HIH', n_header_meta) data = StringIO() data.write(magic) data.write(n_header_meta) data.write(self.getTransport().readAll(sz - 10)) data.seek(10) self.read_header_format(sz - 10, header_size, data) else: self.__client_type = CLIENT_TYPE.UNKNOWN raise TTransportException( TTransportException.INVALID_CLIENT_TYPE, "Could not detect client transport type") if self.__client_type not in self.__supported_client_types: raise TTransportException( TTransportException.INVALID_CLIENT_TYPE, "Client type {} not supported on server".format( self.__client_type)) def read_header_format(self, sz, header_size, data): # clear out any previous transforms self.__read_transforms = [] header_size = header_size * 4 if header_size > sz: raise TTransportException(TTransportException.INVALID_FRAME_SIZE, "Header size is larger than frame") end_header = header_size + data.tell() self.__proto_id = readVarint(data) num_headers = readVarint(data) if self.__proto_id == 1 and self.__client_type != \ CLIENT_TYPE.HTTP_SERVER: raise TTransportException( TTransportException.INVALID_CLIENT_TYPE, "Trying to recv JSON encoding over binary") # Read the headers. Data for each header varies. for _ in range(0, num_headers): trans_id = readVarint(data) if trans_id in (TRANSFORM.ZLIB, TRANSFORM.SNAPPY, TRANSFORM.ZSTD): self.__read_transforms.insert(0, trans_id) elif trans_id == TRANSFORM.HMAC: raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Hmac transform is no longer supported: %i" % trans_id) else: # TApplicationException will be sent back to client raise TApplicationException( TApplicationException.INVALID_TRANSFORM, "Unknown transform in client request: %i" % trans_id) # Clear out previous info headers. self.__read_headers.clear() # Read the info headers. while data.tell() < end_header: info_id = readVarint(data) if info_id == INFO.NORMAL: _read_info_headers(data, end_header, self.__read_headers) elif info_id == INFO.PERSISTENT: _read_info_headers(data, end_header, self.__read_persistent_headers) else: break # Unknown header. Stop info processing. if self.__read_persistent_headers: self.__read_headers.update(self.__read_persistent_headers) # Skip the rest of the header data.seek(end_header) payload = data.read(sz - header_size) # Read the data section. self.__rbuf = StringIO(self.untransform(payload)) def write(self, buf): self.__wbuf.write(buf) def transform(self, buf): for trans_id in self.__write_transforms: if trans_id == TRANSFORM.ZLIB: buf = zlib.compress(buf) elif trans_id == TRANSFORM.SNAPPY: buf = snappy.compress(buf) elif trans_id == TRANSFORM.ZSTD: buf = zstd.ZstdCompressor( write_content_size=True).compress(buf) else: raise TTransportException( TTransportException.INVALID_TRANSFORM, "Unknown transform during send") return buf def untransform(self, buf): for trans_id in self.__read_transforms: if trans_id == TRANSFORM.ZLIB: buf = zlib.decompress(buf) elif trans_id == TRANSFORM.SNAPPY: buf = snappy.decompress(buf) elif trans_id == TRANSFORM.ZSTD: buf = zstd.ZstdDecompressor().decompress(buf) if trans_id not in self.__write_transforms: self.__write_transforms.append(trans_id) return buf def flush(self): self.flushImpl(False) def onewayFlush(self): self.flushImpl(True) def _flushHeaderMessage(self, buf, wout, wsz): """Write a message for CLIENT_TYPE.HEADER @param buf(StringIO): Buffer to write message to @param wout(str): Payload @param wsz(int): Payload length """ transform_data = StringIO() # For now, all transforms don't require data. num_transforms = len(self.__write_transforms) for trans_id in self.__write_transforms: transform_data.write(getVarint(trans_id)) # Add in special flags. if self.__identity: self.__write_headers[self.ID_VERSION_HEADER] = self.ID_VERSION self.__write_headers[self.IDENTITY_HEADER] = self.__identity info_data = StringIO() # Write persistent kv-headers _flush_info_headers(info_data, self.get_write_persistent_headers(), INFO.PERSISTENT) # Write non-persistent kv-headers _flush_info_headers(info_data, self.__write_headers, INFO.NORMAL) header_data = StringIO() header_data.write(getVarint(self.__proto_id)) header_data.write(getVarint(num_transforms)) header_size = transform_data.tell() + header_data.tell() + \ info_data.tell() padding_size = 4 - (header_size % 4) header_size = header_size + padding_size # MAGIC(2) | FLAGS(2) + SEQ_ID(4) + HEADER_SIZE(2) wsz += header_size + 10 if wsz > MAX_FRAME_SIZE: buf.write(pack("!I", BIG_FRAME_MAGIC)) buf.write(pack("!Q", wsz)) else: buf.write(pack("!I", wsz)) buf.write(pack("!HH", HEADER_MAGIC >> 16, self.__flags)) buf.write(pack("!I", self.seq_id)) buf.write(pack("!H", header_size // 4)) buf.write(header_data.getvalue()) buf.write(transform_data.getvalue()) buf.write(info_data.getvalue()) # Pad out the header with 0x00 for _ in range(0, padding_size, 1): buf.write(pack("!c", b'\0')) # Send data section buf.write(wout) def flushImpl(self, oneway): wout = self.__wbuf.getvalue() wout = self.transform(wout) wsz = len(wout) # reset wbuf before write/flush to preserve state on underlying failure self.__wbuf.seek(0) self.__wbuf.truncate() if self.__proto_id == 1 and self.__client_type != CLIENT_TYPE.HTTP_SERVER: raise TTransportException( TTransportException.INVALID_CLIENT_TYPE, "Trying to send JSON encoding over binary") buf = StringIO() if self.__client_type == CLIENT_TYPE.HEADER: self._flushHeaderMessage(buf, wout, wsz) elif self.__client_type in (CLIENT_TYPE.FRAMED_DEPRECATED, CLIENT_TYPE.FRAMED_COMPACT): buf.write(pack("!i", wsz)) buf.write(wout) elif self.__client_type in (CLIENT_TYPE.UNFRAMED_DEPRECATED, CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED): buf.write(wout) elif self.__client_type == CLIENT_TYPE.HTTP_SERVER: # Reset the client type if we sent something - # oneway calls via HTTP expect a status response otherwise buf.write(self.header.getvalue()) buf.write(wout) self.__client_type == CLIENT_TYPE.HEADER elif self.__client_type == CLIENT_TYPE.UNKNOWN: raise TTransportException(TTransportException.INVALID_CLIENT_TYPE, "Unknown client type") # We don't include the framing bytes as part of the frame size check frame_size = buf.tell() - (4 if wsz < MAX_FRAME_SIZE else 12) _frame_size_check(frame_size, self.__max_frame_size, header=self.__client_type == CLIENT_TYPE.HEADER) self.getTransport().write(buf.getvalue()) if oneway: self.getTransport().onewayFlush() else: self.getTransport().flush() # Implement the CReadableTransport interface. @property def cstringio_buf(self): if not self.__rbuf_frame: self.readFrame(0) return self.__rbuf def cstringio_refill(self, prefix, reqlen): # self.__rbuf will already be empty here because fastproto doesn't # ask for a refill until the previous buffer is empty. Therefore, # we can start reading new frames immediately. # On unframed clients, there is a chance there is something left # in rbuf, and the read pointer is not advanced by fastproto # so seek to the end to be safe self.__rbuf.seek(0, 2) while len(prefix) < reqlen: prefix += self.read(reqlen) self.__rbuf = StringIO(prefix) return self.__rbuf
class REL: def __init__(self): self.data = BytesIO() self.id = None self.sections = [] self.name_offset = 0 self.name_length = 0 self.rel_format_version = 3 self.bss_size = 0 self.relocation_entries_for_module = OrderedDict() self.prolog_section = 0 self.epilog_section = 0 self.unresolved_section = 0 self.prolog_offset = 0 self.epilog_offset = 0 self.unresolved_offset = 0 self.alignment = 8 self.bss_alignment = 1 self.fix_size = 0 def read_from_file(self, file_path): with open(file_path, "rb") as file: data = BytesIO(file.read()) self.read(data) def read(self, data): self.data = data data = self.data self.id = read_u32(data, 0) self.sections = [] self.num_sections = read_u32(data, 0xC) self.section_info_table_offset = read_u32(data, 0x10) for section_index in range(0, self.num_sections): section_info_offset = self.section_info_table_offset + section_index * RELSection.ENTRY_SIZE section = RELSection() section.read(data, section_info_offset) self.sections.append(section) self.name_offset = read_u32(data, 0x14) self.name_length = read_u32(data, 0x18) self.rel_format_version = read_u32(data, 0x1C) self.bss_size = read_u32(data, 0x20) relocation_data_offset_for_module = OrderedDict() self.relocation_table_offset = read_u32(data, 0x24) self.imp_table_offset = read_u32(data, 0x28) self.imp_table_length = read_u32(data, 0x2C) offset = self.imp_table_offset while offset < self.imp_table_offset + self.imp_table_length: module_num = read_u32(data, offset) relocation_data_offset = read_u32(data, offset + 4) relocation_data_offset_for_module[ module_num] = relocation_data_offset offset += 8 self.relocation_entries_for_module = OrderedDict() curr_section_num = None for module_num, relocation_data_offset in relocation_data_offset_for_module.items( ): self.relocation_entries_for_module[module_num] = [] offset = relocation_data_offset prev_relocation_offset = 0 while True: relocation_type = RELRelocationType(read_u8(data, offset + 2)) if relocation_type == RELRelocationType.R_DOLPHIN_END: break relocation_data_entry = RELRelocation() relocation_data_entry.read(data, offset, prev_relocation_offset, curr_section_num) prev_relocation_offset = relocation_data_entry.relocation_offset if relocation_data_entry.relocation_type == RELRelocationType.R_DOLPHIN_SECTION: curr_section_num = relocation_data_entry.section_num_to_relocate_against prev_relocation_offset = 0 else: self.relocation_entries_for_module[module_num].append( relocation_data_entry) offset += RELRelocation.ENTRY_SIZE self.prolog_section = read_u8(data, 0x30) self.epilog_section = read_u8(data, 0x31) self.unresolved_section = read_u8(data, 0x32) self.prolog_offset = read_u32(data, 0x34) self.epilog_offset = read_u32(data, 0x38) self.unresolved_offset = read_u32(data, 0x3C) self.alignment = read_u32(data, 0x40) self.bss_alignment = read_u32(data, 0x44) # Space after this fix_size offset can be reused for other purposes. # Such as using the space that originally had the relocations list for .bss static variables instead. self.fix_size = read_u32(data, 0x48) self.bss_section_index = None # The byte at offset 0x33 in the REL is reserved for this value at runtime. for section_index, section in enumerate(self.sections): if section.is_bss: self.bss_section_index = section_index section.offset = self.bss_offset break @property def bss_offset(self): # BSS doesn't start until the next 0x20 byte alignment after the end of the initialized data (specified by fix_size). return (self.fix_size + 0x1F) & ~(0x1F) def convert_rel_offset_to_section_index_and_relative_offset(self, offset): section_index = None relative_offset = None for section in self.sections: if section.is_uninitialized: continue if section.offset <= offset < section.offset + section.length: section_index = self.sections.index(section) relative_offset = offset - section.offset break if section_index is None: raise Exception( "Offset %04X is not in the data for any of the REL sections" % offset) return (section_index, relative_offset) def convert_rel_offset_to_section_data_and_relative_offset(self, offset): data = None relative_offset = None for section in self.sections: if section.is_uninitialized: continue if section.offset <= offset < section.offset + section.length: data = section.data relative_offset = offset - section.offset break return (data, relative_offset) def read_data(self, read_callback, offset, *args): # This function allows reading from a REL using the offset from the start of the REL file, instead of needing to know the section index and offset within that section. data, relative_offset = self.convert_rel_offset_to_section_data_and_relative_offset( offset) if data is None: raise Exception( "Offset %04X is not in the data for any of the REL sections" % offset) return read_callback(data, relative_offset, *args) def write_data(self, write_callback, offset, *args, delete_relocations=False): # This function allows writing to a REL using the offset from the start of the REL file, instead of needing to know the section index and offset within that section. data, relative_offset = self.convert_rel_offset_to_section_data_and_relative_offset( offset) if data is None: raise Exception( "Offset %04X is not in the data for any of the REL sections" % offset) write_callback(data, relative_offset, *args) def delete_relocation_in_range(self, offset, length): for module_num, relocations in self.relocation_entries_for_module.items( ): relocations_to_delete = [] for relocation in relocations: curr_section = self.sections[relocation.curr_section_num] relocation_absolute_offset = relocation.relocation_offset + curr_section.offset if relocation_absolute_offset >= offset and relocation_absolute_offset < offset + length: relocations_to_delete.append(relocation) for relocation in relocations_to_delete: relocations.remove(relocation) def save_to_file(self, file_path, preserve_section_data_offsets=False): self.save_changes( preserve_section_data_offsets=preserve_section_data_offsets) with open(file_path, "wb") as f: f.write(read_all_bytes(self.data)) def save_changes(self, preserve_section_data_offsets=False): self.data.truncate(0) data = self.data write_u32(data, 0x00, self.id) write_u32(data, 0x04, 0) write_u32(data, 0x08, 0) self.num_sections = len(self.sections) write_u32(data, 0x0C, self.num_sections) write_u32(data, 0x14, self.name_offset) write_u32(data, 0x18, self.name_length) write_u32(data, 0x1C, self.rel_format_version) write_u32( data, 0x20, self.bss_size) # TODO recalculate this properly when necessary self.section_info_table_offset = 0x4C write_u32(data, 0x10, self.section_info_table_offset) next_section_info_offset = self.section_info_table_offset next_section_data_offset = self.section_info_table_offset + self.num_sections * RELSection.ENTRY_SIZE next_section_data_offset = pad_offset_to_nearest( next_section_data_offset, 4 ) # TODO why is 4 more accurate here than the 8 from self.alignment? for section_index, section in enumerate(self.sections): if preserve_section_data_offsets: if section.is_uninitialized: # An uninitialized section. # We don't need to preserve the data offsets for these. Do nothing. pass else: assert section.offset >= next_section_data_offset next_section_data_offset = section.offset section.save(data, next_section_info_offset, next_section_data_offset, self.bss_size) next_section_info_offset += RELSection.ENTRY_SIZE if not section.is_bss: next_section_data_offset += section.length next_section_data_offset = pad_offset_to_nearest( next_section_data_offset, 4) # We need to reorder the relocations list before writing it so that relocations against this current REL and relocations against main come after relocations against other RELs. # This is because the game assumes those two are always last, and shrinks the size of the imp table to not include those and anything after them upon first relocation (because those two are guaranteed to be complete on first relocation, unlike relocations against any other REL). if self.id in self.relocation_entries_for_module: relocations_against_this_rel = self.relocation_entries_for_module.pop( self.id) self.relocation_entries_for_module[ self.id] = relocations_against_this_rel if 0 in self.relocation_entries_for_module: relocations_against_main = self.relocation_entries_for_module.pop( 0) self.relocation_entries_for_module[0] = relocations_against_main self.imp_table_offset = data_len(data) imp_table_size = len(self.relocation_entries_for_module) * 8 imp_table_end = self.imp_table_offset + imp_table_size self.relocation_table_offset = imp_table_end self.fix_size = self.relocation_table_offset write_u32(data, 0x24, self.relocation_table_offset) write_u32(data, 0x28, self.imp_table_offset) write_u32(data, 0x2C, imp_table_size) next_imp_offset = self.imp_table_offset next_relocation_entry_offset = self.relocation_table_offset for module_num, relocation_data_entries in self.relocation_entries_for_module.items( ): write_u32(data, next_imp_offset + 0x00, module_num) write_u32(data, next_imp_offset + 0x04, next_relocation_entry_offset) next_imp_offset += 8 # Sort the relocations first by their section, then by their offset within the section. relocation_data_entries.sort(key=lambda reloc: ( reloc.curr_section_num, reloc.relocation_offset)) curr_section_num = None prev_relocation_offset = 0 for relocation_data_entry in relocation_data_entries: if relocation_data_entry.curr_section_num != curr_section_num: curr_section_num = relocation_data_entry.curr_section_num prev_relocation_offset = 0 section_start_entry = RELRelocation() section_start_entry.relocation_type = RELRelocationType.R_DOLPHIN_SECTION section_start_entry.section_num_to_relocate_against = curr_section_num section_start_entry.save(data, next_relocation_entry_offset) next_relocation_entry_offset += RELRelocation.ENTRY_SIZE offset_diff = relocation_data_entry.relocation_offset - prev_relocation_offset while offset_diff > 0xFFFF: # The offset change is stored as a halfword, so if the gap is too large, we must insert a NOP command (or several) to bridge the gap. nop_entry = RELRelocation() nop_entry.relocation_type = RELRelocationType.R_DOLPHIN_NOP relocation_data_entry.relocation_offset = prev_relocation_offset + 0xFFFF relocation_data_entry.offset_of_curr_relocation_from_prev = 0xFFFF nop_entry.save(data, next_relocation_entry_offset) prev_relocation_offset = relocation_data_entry.relocation_offset next_relocation_entry_offset += RELRelocation.ENTRY_SIZE offset_diff -= 0xFFFF if offset_diff < 0: raise Exception( "Negative offset difference between relocation. Relocations not properly sorted." ) relocation_data_entry.offset_of_curr_relocation_from_prev = offset_diff relocation_data_entry.save(data, next_relocation_entry_offset) prev_relocation_offset = relocation_data_entry.relocation_offset next_relocation_entry_offset += RELRelocation.ENTRY_SIZE table_end_entry = RELRelocation() table_end_entry.relocation_type = RELRelocationType.R_DOLPHIN_END table_end_entry.section_num_to_relocate_against = curr_section_num table_end_entry.save(data, next_relocation_entry_offset) next_relocation_entry_offset += RELRelocation.ENTRY_SIZE if module_num != 0 and module_num != self.id: # Normally fix_size wouldn't need to include any of the relocation table in it, because all relocations happen when the REL is first loaded, and the whole relocation table can be repurposed afterwards. # But when the REL has relocations against a module besides main.dol and itself, that is no longer the case. # Relocations against a different REL can happen after this REL is initially loaded, so we need to include those relocations in fix_size. # Only relocations after the end of the last REL-to-REL relocation can be repurposed. self.fix_size = next_relocation_entry_offset write_u8(data, 0x30, self.prolog_section) write_u8(data, 0x31, self.epilog_section) write_u8(data, 0x32, self.unresolved_section) write_u32(data, 0x34, self.prolog_offset) write_u32(data, 0x38, self.epilog_offset) write_u32(data, 0x3C, self.unresolved_offset) write_u32(data, 0x40, self.alignment) write_u32(data, 0x44, self.bss_alignment) # TODO: align bss to the bss_alignment write_u32(data, 0x48, self.fix_size)
def captureMotion(): global motionMonitorStream, motionDetected, camera, cfgPreMotionCapture, cfgPostMotionCapture global cfgDeviceEnabled global currentlyRecording # Max length of video to capture streamMaxSeconds = 60 # Time between checking for motion timeBetweenMotionChecks = 5 # While keeps loop running while True: # Check if we should not detect motion if not cfgDeviceEnabled: logDebug("Device not enabled") elif motionDetected: logDebug("Starting recording of motion") currentlyRecording = True # Copy x amount of seconds to a new BytesIO buffer and then split recording immediately after to avoid # delay in video recordingStream = BytesIO() motionMonitorStream.copy_to(recordingStream, seconds=cfgPreMotionCapture) camera.split_recording(recordingStream) # Clean up motion monitor and setup time motionMonitorStream.clear() # Setup recording time recordingTime = 0 + cfgPreMotionCapture + cfgPostMotionCapture # Capture thumbnail thumbnailStream = BytesIO() camera.capture(thumbnailStream, format='jpeg', use_video_port=True, resize=(320, 240)) thumbnailStream.seek(0) # The idea here is to keep recording while we detect motion or until we run out of room to record # The only way to exit this loop is after we wend a recording while True: sendRecording = False if motionDetected: logDebug("Recorded total of " + str(recordingTime) + " seconds") # Reset motion, then sleep, if it happens again then we will keep recording motionDetected = False # VERY IMPORTANT! Must exist before sleep, otherwise you will set it and then quickly evaluate it camera.wait_recording( timeBetweenMotionChecks ) # If set too low, you may get more choppy videos motion may not be detected within 1 second recordingTime = recordingTime + timeBetweenMotionChecks # If we go beyond the time we have available, then we need to send the recording out and start new # Uploading 30 second chunks at at time sends a manageable amount to the server if recordingTime > streamMaxSeconds: logDebug("Reached max recording at " + str(recordingTime) + " seconds, starting new") sendRecording = True else: logDebug("Motion no longer detected at " + str(recordingTime) + " seconds, sending capture") sendRecording = True currentlyRecording = False if sendRecording: logDebug("Sending Recording of " + str(recordingTime) + " seconds") # Capture the post motion camera.wait_recording(cfgPostMotionCapture) # Copy recording stream to transferToServerStream transferToServerStream = BytesIO() recordingStream.seek(0) transferToServerStream.write(recordingStream.read()) transferToServerStream.seek(0) # Jump back to motionMonitorStream briefly to clear recording stream camera.split_recording(motionMonitorStream) # Clear our recording stream by seeking to 0 and then truncate recordingStream.seek(0) recordingStream.truncate() # If we are still recording, we need to jump back to recording stream if currentlyRecording: camera.split_recording(recordingStream) # Send the video to the server t = threading.Thread(target=postVideo, args=(transferToServerStream, recordingTime, thumbnailStream)) t.start() # If we are still recording, we should continue on, otherwise we should break if currentlyRecording: continue else: break # Wait one second until checking again and reset motion motionDetected = False # This is reset in case motion was detected but the camera was not enabled sleep(1)
def run(self): #results = {} # {'file' : error_code,...} STATE_DONE = 0 STATE_ABORTED = 10 STATE_SUCCESS = 20 STATE_BUSY = 25 STATE_READ_SENDER_INFO = 30 STATE_PRERENDER = 40 STATE_COUNT_PAGES = 50 STATE_NEXT_RECIPIENT = 60 STATE_COVER_PAGE = 70 STATE_SINGLE_FILE = 80 STATE_MERGE_FILES = 90 STATE_SINGLE_FILE = 100 STATE_SEND_FAX = 110 STATE_CLEANUP = 120 STATE_ERROR = 130 next_recipient = self.next_recipient_gen() state = STATE_READ_SENDER_INFO self.rendered_file_list = [] while state != STATE_DONE: # --------------------------------- Fax state machine if self.check_for_cancel(): state = STATE_ABORTED log.debug("STATE=(%d, 0, 0)" % state) if state == STATE_ABORTED: # --------------------------------- Aborted (10, 0, 0) log.error("Aborted by user.") self.write_queue((STATUS_IDLE, 0, '')) state = STATE_CLEANUP elif state == STATE_SUCCESS: # --------------------------------- Success (20, 0, 0) log.debug("Success.") self.write_queue((STATUS_COMPLETED, 0, '')) state = STATE_CLEANUP elif state == STATE_ERROR: # --------------------------------- Error (130, 0, 0) log.error("Error, aborting.") self.write_queue((STATUS_ERROR, 0, '')) state = STATE_CLEANUP elif state == STATE_BUSY: # --------------------------------- Busy (25, 0, 0) log.error("Device busy, aborting.") self.write_queue((STATUS_BUSY, 0, '')) state = STATE_CLEANUP elif state == STATE_READ_SENDER_INFO: # --------------------------------- Get sender info (30, 0, 0) log.debug("%s State: Get sender info" % ("*" * 20)) state = STATE_PRERENDER try: try: self.dev.open() except Error as e: log.error("Unable to open device (%s)." % e.msg) state = STATE_ERROR else: try: self.sender_name = self.dev.station_name log.debug("Sender name=%s" % self.sender_name) self.sender_fax = self.dev.phone_num log.debug("Sender fax=%s" % self.sender_fax) except Error: log.error("PML get failed!") state = STATE_ERROR finally: self.dev.close() elif state == STATE_PRERENDER: # --------------------------------- Pre-render non-G3 files (40, 0, 0) log.debug("%s State: Pre-render non-G3 files" % ("*" * 20)) state = self.pre_render(STATE_COUNT_PAGES) elif state == STATE_COUNT_PAGES: # --------------------------------- Get total page count (50, 0, 0) log.debug("%s State: Get total page count" % ("*" * 20)) state = self.count_pages(STATE_NEXT_RECIPIENT) elif state == STATE_NEXT_RECIPIENT: # --------------------------------- Loop for multiple recipients (60, 0, 0) log.debug("%s State: Next recipient" % ("*" * 20)) state = STATE_COVER_PAGE try: recipient = next(next_recipient) #print recipient log.debug("Processing for recipient %s" % recipient['name']) self.write_queue( (STATUS_SENDING_TO_RECIPIENT, 0, recipient['name'])) except StopIteration: state = STATE_SUCCESS log.debug("Last recipient.") continue self.recipient_file_list = self.rendered_file_list[:] elif state == STATE_COVER_PAGE: # --------------------------------- Create cover page (70, 0, 0) log.debug("%s State: Render cover page" % ("*" * 20)) state = self.cover_page(recipient) elif state == STATE_SINGLE_FILE: # --------------------------------- Special case for single file (no merge) (80, 0, 0) log.debug("%s State: Handle single file" % ("*" * 20)) state = self.single_file(STATE_SEND_FAX) elif state == STATE_MERGE_FILES: # --------------------------------- Merge multiple G3 files (90, 0, 0) log.debug("%s State: Merge multiple files" % ("*" * 20)) state = self.merge_files(STATE_SEND_FAX) elif state == STATE_SEND_FAX: # --------------------------------- Send fax state machine (110, 0, 0) log.debug("%s State: Send fax" % ("*" * 20)) state = STATE_NEXT_RECIPIENT FAX_SEND_STATE_DONE = 0 FAX_SEND_STATE_ABORT = 10 FAX_SEND_STATE_ERROR = 20 FAX_SEND_STATE_BUSY = 25 FAX_SEND_STATE_SUCCESS = 30 FAX_SEND_STATE_DEVICE_OPEN = 40 FAX_SEND_STATE_SET_TOKEN = 50 FAX_SEND_STATE_EARLY_OPEN = 60 FAX_SEND_STATE_SET_PARAMS = 70 FAX_SEND_STATE_CHECK_IDLE = 80 FAX_SEND_STATE_START_REQUEST = 90 FAX_SEND_STATE_LATE_OPEN = 100 FAX_SEND_STATE_SEND_DIAL_STRINGS = 110 FAX_SEND_STATE_SEND_FAX_HEADER = 120 FAX_SEND_STATE_SEND_PAGES = 130 FAX_SEND_STATE_SEND_END_OF_STREAM = 140 FAX_SEND_STATE_WAIT_FOR_COMPLETE = 150 FAX_SEND_STATE_RESET_TOKEN = 160 FAX_SEND_STATE_CLOSE_SESSION = 170 monitor_state = False error_state = pml.DN_ERROR_NONE fax_send_state = FAX_SEND_STATE_DEVICE_OPEN while fax_send_state != FAX_SEND_STATE_DONE: if self.check_for_cancel(): log.error("Fax send aborted.") fax_send_state = FAX_SEND_STATE_ABORT if monitor_state: fax_state = self.getFaxDownloadState() if not fax_state in (pml.UPDN_STATE_XFERACTIVE, pml.UPDN_STATE_XFERDONE): log.error("D/L error state=%d" % fax_state) fax_send_state = FAX_SEND_STATE_ERROR state = STATE_ERROR log.debug("STATE=(%d, %d, 0)" % (STATE_SEND_FAX, fax_send_state)) if fax_send_state == FAX_SEND_STATE_ABORT: # -------------- Abort (110, 10, 0) # TODO: Set D/L state to ??? monitor_state = False fax_send_state = FAX_SEND_STATE_RESET_TOKEN state = STATE_ABORTED elif fax_send_state == FAX_SEND_STATE_ERROR: # -------------- Error (110, 20, 0) log.error("Fax send error.") error_state = self.getFaxDownloadError() log.debug( "Error State=%d (%s)" % (error_state, pml.DN_ERROR_STR.get(error_state, "Unknown"))) monitor_state = False fax_send_state = FAX_SEND_STATE_RESET_TOKEN state = STATE_ERROR elif fax_send_state == FAX_SEND_STATE_BUSY: # -------------- Busy (110, 25, 0) log.error("Fax device busy.") monitor_state = False fax_send_state = FAX_SEND_STATE_RESET_TOKEN state = STATE_BUSY elif fax_send_state == FAX_SEND_STATE_SUCCESS: # -------------- Success (110, 30, 0) log.debug("Fax send success.") monitor_state = False fax_send_state = FAX_SEND_STATE_RESET_TOKEN state = STATE_NEXT_RECIPIENT elif fax_send_state == FAX_SEND_STATE_DEVICE_OPEN: # -------------- Device open (110, 40, 0) log.debug("%s State: Open device" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SET_TOKEN try: self.dev.open() except Error as e: log.error("Unable to open device (%s)." % e.msg) fax_send_state = FAX_SEND_STATE_ERROR else: if self.dev.device_state == DEVICE_STATE_NOT_FOUND: fax_send_state = FAX_SEND_STATE_ERROR elif fax_send_state == FAX_SEND_STATE_SET_TOKEN: # -------------- Acquire fax token (110, 50, 0) log.debug("%s State: Acquire fax token" % ("*" * 20)) try: result_code, token = self.dev.getPML( pml.OID_FAX_TOKEN) except Error: log.debug("Unable to acquire fax token (1).") fax_send_state = FAX_SEND_STATE_EARLY_OPEN else: if result_code > pml.ERROR_MAX_OK: fax_send_state = FAX_SEND_STATE_EARLY_OPEN log.debug("Skipping token acquisition.") else: token = time.strftime("%d%m%Y%H:%M:%S", time.gmtime()) log.debug("Setting token: %s" % token) try: self.dev.setPML(pml.OID_FAX_TOKEN, token) except Error: log.error( "Unable to acquire fax token (2).") fax_send_state = FAX_SEND_STATE_ERROR else: result_code, check_token = self.dev.getPML( pml.OID_FAX_TOKEN) if check_token == token: fax_send_state = FAX_SEND_STATE_EARLY_OPEN else: log.error( "Unable to acquire fax token (3).") fax_send_state = FAX_SEND_STATE_ERROR elif fax_send_state == FAX_SEND_STATE_EARLY_OPEN: # -------------- Early open (newer models) (110, 60, 0) log.debug("%s State: Early open" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_CHECK_IDLE if self.dev.fax_type == FAX_TYPE_BLACK_SEND_EARLY_OPEN: # newer log.debug("Opening fax channel.") try: self.dev.openFax() except Error as e: log.error("Unable to open channel (%s)." % e.msg) fax_send_state = FAX_SEND_STATE_ERROR else: log.debug("Skipped.") elif fax_send_state == FAX_SEND_STATE_CHECK_IDLE: # -------------- Check for initial idle (110, 80, 0) log.debug("%s State: Check idle" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_START_REQUEST dl_state = self.getFaxDownloadState() tx_status = self.getFaxJobTxStatus() rx_status = self.getFaxJobRxStatus() if ((dl_state == pml.UPDN_STATE_IDLE or \ dl_state == pml.UPDN_STATE_ERRORABORT or \ dl_state == pml.UPDN_STATE_XFERDONE) and \ (tx_status == pml.FAXJOB_TX_STATUS_IDLE or tx_status == pml.FAXJOB_TX_STATUS_DONE) and \ (rx_status == pml.FAXJOB_RX_STATUS_IDLE or rx_status == pml.FAXJOB_RX_STATUS_DONE)): # xwas if state == pml.UPDN_STATE_IDLE: if dl_state == pml.UPDN_STATE_IDLE: log.debug("Starting in idle state") else: log.debug("Resetting to idle...") self.dev.setPML(pml.OID_FAX_DOWNLOAD, pml.UPDN_STATE_IDLE) time.sleep(0.5) else: fax_send_state = FAX_SEND_STATE_BUSY elif fax_send_state == FAX_SEND_STATE_START_REQUEST: # -------------- Request fax start (110, 90, 0) log.debug("%s State: Request start" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SET_PARAMS dl_state = self.getFaxDownloadState() if dl_state == pml.UPDN_STATE_IDLE: log.debug("Try: 0") log.debug( "Setting to up/down state request start...") self.dev.setPML(pml.OID_FAX_DOWNLOAD, pml.UPDN_STATE_REQSTART) time.sleep(1) log.debug("Waiting for active state...") i = 1 while i < 10: log.debug("Try: %d" % i) try: dl_state = self.getFaxDownloadState() except Error: log.error("PML/SNMP error") fax_send_state = FAX_SEND_STATE_ERROR break if dl_state == pml.UPDN_STATE_XFERACTIVE: break time.sleep(1) log.debug( "Setting to up/down state request start..." ) self.dev.setPML(pml.OID_FAX_DOWNLOAD, pml.UPDN_STATE_REQSTART) i += 1 else: log.error("Could not get into active state!") fax_send_state = FAX_SEND_STATE_BUSY monitor_state = True else: log.error("Could not get into idle state!") fax_send_state = FAX_SEND_STATE_BUSY elif fax_send_state == FAX_SEND_STATE_SET_PARAMS: # -------------- Set fax send params (110, 70, 0) log.debug("%s State: Set params" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_LATE_OPEN try: self.dev.setPML(pml.OID_DEV_DOWNLOAD_TIMEOUT, pml.DEFAULT_DOWNLOAD_TIMEOUT) self.dev.setPML(pml.OID_FAXJOB_TX_TYPE, pml.FAXJOB_TX_TYPE_HOST_ONLY) log.debug("Setting date and time on device.") self.dev.setDateAndTime() except Error as e: log.error("PML/SNMP error (%s)" % e.msg) fax_send_state = FAX_SEND_STATE_ERROR elif fax_send_state == FAX_SEND_STATE_LATE_OPEN: # -------------- Late open (older models) (110, 100, 0) log.debug("%s State: Late open" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SEND_DIAL_STRINGS if self.dev.fax_type == FAX_TYPE_BLACK_SEND_LATE_OPEN: # older log.debug("Opening fax channel.") try: self.dev.openFax() except Error: log.error("Unable to open channel.") fax_send_state = FAX_SEND_STATE_ERROR else: log.debug("Skipped.") elif fax_send_state == FAX_SEND_STATE_SEND_DIAL_STRINGS: # -------------- Dial strings (110, 110, 0) log.debug("%s State: Send dial strings" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SEND_FAX_HEADER log.debug("Dialing: %s" % recipient['fax']) log.debug("Sending dial strings...") self.create_mfpdtf_fixed_header( DT_DIAL_STRINGS, True, PAGE_FLAG_NEW_DOC | PAGE_FLAG_END_DOC | PAGE_FLAG_END_STREAM ) # 0x1c on Windows, we were sending 0x0c #print recipient dial_strings = recipient['fax'].encode('ascii') log.debug(repr(dial_strings)) self.create_mfpdtf_dial_strings(dial_strings) try: self.write_stream() except Error: log.error("Channel write error.") fax_send_state = FAX_SEND_STATE_ERROR elif fax_send_state == FAX_SEND_STATE_SEND_FAX_HEADER: # -------------- Fax header (110, 120, 0) log.debug("%s State: Send fax header" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SEND_PAGES try: ff = open(self.f, 'rb') except IOError: log.error("Unable to read fax file.") fax_send_state = FAX_SEND_STATE_ERROR continue try: header = ff.read(FILE_HEADER_SIZE) except IOError: log.error("Unable to read fax file.") fax_send_state = FAX_SEND_STATE_ERROR continue magic, version, total_pages, hort_dpi, vert_dpi, page_size, \ resolution, encoding, reserved1, reserved2 = self.decode_fax_header(header) if magic != b'hplip_g3': log.error("Invalid file header. Bad magic.") fax_send_state = FAX_SEND_STATE_ERROR else: log.debug( "Magic=%s Ver=%d Pages=%d hDPI=%d vDPI=%d Size=%d Res=%d Enc=%d" % (magic, version, total_pages, hort_dpi, vert_dpi, page_size, resolution, encoding)) log.debug("Sending fax header...") self.create_mfpdtf_fixed_header( DT_FAX_IMAGES, True, PAGE_FLAG_NEW_DOC) self.create_mfpdtf_fax_header(total_pages) try: self.write_stream() except Error: log.error("Unable to write to channel.") fax_send_state = FAX_SEND_STATE_ERROR elif fax_send_state == FAX_SEND_STATE_SEND_PAGES: # --------------------------------- Send fax pages state machine (110, 130, 0) log.debug("%s State: Send pages" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_SEND_END_OF_STREAM page = BytesIO() for p in range(total_pages): if self.check_for_cancel(): fax_send_state = FAX_SEND_STATE_ABORT if fax_send_state == FAX_SEND_STATE_ABORT: break try: header = ff.read(PAGE_HEADER_SIZE) except IOError: log.error("Unable to read fax file.") fax_send_state = FAX_SEND_STATE_ERROR continue page_num, ppr, rpp, bytes_to_read, thumbnail_bytes, reserved2 = \ self.decode_page_header(header) log.debug("Page=%d PPR=%d RPP=%d BPP=%d Thumb=%d" % (page_num, ppr, rpp, bytes_to_read, thumbnail_bytes)) page.write(ff.read(bytes_to_read)) thumbnail = ff.read( thumbnail_bytes ) # thrown away for now (should be 0 read) page.seek(0) self.create_mfpdtf_fixed_header( DT_FAX_IMAGES, page_flags=PAGE_FLAG_NEW_PAGE) self.create_sop_record(page_num, hort_dpi, vert_dpi, ppr, rpp, encoding) try: data = page.read(RASTER_DATA_SIZE) except IOError: log.error("Unable to read fax file.") fax_send_state = FAX_SEND_STATE_ERROR continue if data == '': log.error("No data!") fax_send_state = FAX_SEND_STATE_ERROR continue self.create_raster_data_record(data) total_read = RASTER_DATA_SIZE while True: data = page.read(RASTER_DATA_SIZE) total_read += RASTER_DATA_SIZE dl_state = self.getFaxDownloadState() if dl_state == pml.UPDN_STATE_ERRORABORT: fax_send_state = FAX_SEND_STATE_ERROR break if self.check_for_cancel(): fax_send_state = FAX_SEND_STATE_ABORT break if data == b'': self.create_eop_record(rpp) try: self.write_stream() except Error: log.error("Channel write error.") fax_send_state = FAX_SEND_STATE_ERROR break else: try: self.write_stream() except Error: log.error("Channel write error.") fax_send_state = FAX_SEND_STATE_ERROR break status = self.getFaxJobTxStatus() while status == pml.FAXJOB_TX_STATUS_DIALING: self.write_queue( (STATUS_DIALING, 0, recipient['fax'])) time.sleep(1.0) if self.check_for_cancel(): fax_send_state = FAX_SEND_STATE_ABORT break dl_state = self.getFaxDownloadState() if dl_state == pml.UPDN_STATE_ERRORABORT: fax_send_state = FAX_SEND_STATE_ERROR break status = self.getFaxJobTxStatus() if fax_send_state not in ( FAX_SEND_STATE_ABORT, FAX_SEND_STATE_ERROR): while status == pml.FAXJOB_TX_STATUS_CONNECTING: self.write_queue((STATUS_CONNECTING, 0, recipient['fax'])) time.sleep(1.0) if self.check_for_cancel(): fax_send_state = FAX_SEND_STATE_ABORT break dl_state = self.getFaxDownloadState() if dl_state == pml.UPDN_STATE_ERRORABORT: fax_send_state = FAX_SEND_STATE_ERROR break status = self.getFaxJobTxStatus() if status == pml.FAXJOB_TX_STATUS_TRANSMITTING: self.write_queue((STATUS_SENDING, page_num, recipient['fax'])) self.create_mfpdtf_fixed_header(DT_FAX_IMAGES, page_flags=0) self.create_raster_data_record(data) if fax_send_state in (FAX_SEND_STATE_ABORT, FAX_SEND_STATE_ERROR): break page.truncate(0) page.seek(0) elif fax_send_state == FAX_SEND_STATE_SEND_END_OF_STREAM: # -------------- EOS (110, 140, 0) log.debug("%s State: Send EOS" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_WAIT_FOR_COMPLETE log.debug("End of stream...") self.create_mfpdtf_fixed_header( DT_FAX_IMAGES, False, PAGE_FLAG_END_STREAM) try: self.write_stream() except Error: log.error("Channel write error.") fax_send_state = FAX_SEND_STATE_ERROR monitor_state = False elif fax_send_state == FAX_SEND_STATE_WAIT_FOR_COMPLETE: # -------------- Wait for complete (110, 150, 0) log.debug("%s State: Wait for completion" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_WAIT_FOR_COMPLETE time.sleep(1.0) status = self.getFaxJobTxStatus() if status == pml.FAXJOB_TX_STATUS_DIALING: self.write_queue( (STATUS_DIALING, 0, recipient['fax'])) log.debug("Dialing ...") elif status == pml.FAXJOB_TX_STATUS_TRANSMITTING: self.write_queue( (STATUS_SENDING, page_num, recipient['fax'])) log.debug("Transmitting ...") elif status in (pml.FAXJOB_TX_STATUS_DONE, pml.FAXJOB_RX_STATUS_IDLE): fax_send_state = FAX_SEND_STATE_RESET_TOKEN state = STATE_NEXT_RECIPIENT log.debug("Transmitting done or idle ...") else: self.write_queue( (STATUS_SENDING, page_num, recipient['fax'])) log.debug("Pending ...") elif fax_send_state == FAX_SEND_STATE_RESET_TOKEN: # -------------- Release fax token (110, 160, 0) log.debug("%s State: Release fax token" % ("*" * 20)) self.write_queue((STATUS_CLEANUP, 0, '')) try: self.dev.setPML(pml.OID_FAX_TOKEN, '\x00' * 16) except Error: log.error("Unable to release fax token.") fax_send_state = FAX_SEND_STATE_CLOSE_SESSION elif fax_send_state == FAX_SEND_STATE_CLOSE_SESSION: # -------------- Close session (110, 170, 0) log.debug("%s State: Close session" % ("*" * 20)) fax_send_state = FAX_SEND_STATE_DONE log.debug("Closing session...") try: mm.close() except NameError: pass try: ff.close() except NameError: pass if self.dev.fax_type == FAX_TYPE_BLACK_SEND_LATE_OPEN: log.debug("Closing fax channel.") self.dev.closeFax() self.dev.setPML(pml.OID_FAX_DOWNLOAD, pml.UPDN_STATE_IDLE) time.sleep(1) if self.dev.fax_type == FAX_TYPE_BLACK_SEND_EARLY_OPEN: log.debug("Closing fax channel.") self.dev.closeFax() self.dev.close() elif state == STATE_CLEANUP: # --------------------------------- Cleanup (120, 0, 0) log.debug("%s State: Cleanup" % ("*" * 20)) if self.remove_temp_file: log.debug("Removing merged file: %s" % self.f) try: os.remove(self.f) log.debug("Removed") except OSError: log.debug("Not found") state = STATE_DONE
class FP_TranslatorRestAPI: logLevel = 1 translatorUrl = 'http://oldTranslator.com:8580/translator/services/FormatBuilder' translatorUrl = 'http://newTranslator.com:8380/translator/services/FormatBuilder' # Mostly flight plans will be passed in as xml strings in inString def __init__(self, inFile=None, outFile=None, url=None, inString=None): if url is None: self.url = FP_TranslatorRestAPI.translatorUrl else: self.url = url if inFile is None: if inString is None: self.log( "No input translator message given, stopping FP_TranslatorRestAPI", 1) sys.exit() else: self.EF_req_data = inString else: self.inFile = inFile with open(self.inFile) as inFileH: self.EF_req_data = inFileH.read() inFileH.close() # gets outFile squared away if outFile is not None: self.outFile = outFile self.buffer = BytesIO() headers = 'SOAPAction: Header|Accept: application/json' self.hdrs = headers.split("|") self.soapEnv = '''<?xml version="1.0" encoding="UTF-8"?> <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> <soapenv:Body>%s</soapenv:Body> </soapenv:Envelope>''' self.body = self.soapEnv % self.EF_req_data self.elapsed_time = None # returns the translator rquest message as string, no soap envelope def getEF003_ReqBody(self): return self.EF_req_data def getElapsedTime(self): return self.elapsed_time # sets the translator request message, and puts in soap envelope def setBody(self, body): self.body = self.soapEnv % body def setBodyFromFile(self, inFile): with open(self.inFile) as inFileH: body = inFileH.read() inFileH.close() self.body = self.soapEnv % body def writeRespToFile(self, body=None, respFile=None): if body is None: print( "FP_TranslatorRestAPI: message body not included with Write request." ) return if respFile is None: print( "FP_TranslatorRestAPI: new file name not included with Write request." ) return with open(respFile, 'w') as outFileH: outFileH.write(body) outFileH.close() def sendFlightPlan(self): # sets up curl object c = pycurl.Curl() c.setopt(pycurl.HTTPHEADER, self.hdrs) c.setopt(pycurl.POST, 1) c.setopt(pycurl.VERBOSE, 1) c.setopt(pycurl.URL, self.url) c.setopt(pycurl.POSTFIELDS, self.body) c.setopt(c.WRITEDATA, self.buffer) # sets up holder for curl command response self.buffer.truncate(0) self.buffer.seek(0) # runs the curl command sent_time = time.time() c.perform() self.elapsed_time = time.time() - sent_time status = c.getinfo(pycurl.RESPONSE_CODE) if status != 200: self.log("\n***** PYCURL HTTP STATUS CODE: %d *****\n" % status, 2) del c # extracts the response as string, from the soap envelope result = self.buffer.getvalue().decode('iso-8859-1') body = result[result.find("<xml_Flight_Plan"):] body = body[:body.find("</soapenv:Body>")] # writes the EF003 response as a file locally for someone to play with # if we didn't get a response, this writes the sent body to the file. # returns the EF003 response to the user As String return body def log(self, text, level=5): if FP_TranslatorRestAPI.logLevel >= level: print(text)
class CSVLogger(_Logger): """ CSV output, consisting of one line per entry. Entries are separated by a separator (a semicolon per default). """ LoggerName = "csv" LoggerArgs = { "filename": "linkchecker-out.csv", 'separator': ';', "quotechar": '"', "dialect": "excel", } def __init__(self, **kwargs): """Store default separator and (os dependent) line terminator.""" args = self.get_args(kwargs) super(CSVLogger, self).__init__(**args) self.init_fileoutput(args) self.separator = args['separator'] self.quotechar = args['quotechar'] self.dialect = args['dialect'] self.linesep = os.linesep def comment(self, s, **args): """Write CSV comment.""" self.writeln(s=u"# %s" % s, **args) def start_output(self): """Write checking start info as csv comment.""" super(CSVLogger, self).start_output() row = [] if self.has_part("intro"): self.write_intro() self.flush() else: # write empty string to initialize file output self.write(u"") self.queue = BytesIO() self.writer = csv.writer(self.queue, dialect=self.dialect, delimiter=self.separator, lineterminator=self.linesep, quotechar=self.quotechar) for s in Columns: if self.has_part(s): row.append(s) if row: self.writerow(row) def log_url(self, url_data): """Write csv formatted url check info.""" row = [] if self.has_part("urlname"): row.append(url_data.base_url) if self.has_part("parentname"): row.append(url_data.parent_url) if self.has_part("baseref"): row.append(url_data.base_ref) if self.has_part("result"): row.append(url_data.result) if self.has_part("warningstring"): row.append(self.linesep.join(x[1] for x in url_data.warnings)) if self.has_part("infostring"): row.append(self.linesep.join(url_data.info)) if self.has_part("valid"): row.append(url_data.valid) if self.has_part("url"): row.append(url_data.url) if self.has_part("line"): row.append(url_data.line) if self.has_part("column"): row.append(url_data.column) if self.has_part("name"): row.append(url_data.name) if self.has_part("dltime"): row.append(url_data.dltime) if self.has_part("dlsize"): row.append(url_data.size) if self.has_part("checktime"): row.append(url_data.checktime) if self.has_part("cached"): row.append(0) if self.has_part("level"): row.append(url_data.level) if self.has_part("modified"): row.append(self.format_modified(url_data.modified)) self.writerow(map(strformat.unicode_safe, row)) self.flush() def writerow(self, row): """Write one row in CSV format.""" self.writer.writerow( [s.encode("utf-8", self.codec_errors) for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and write to the target stream self.write(data) # empty queue self.queue.truncate(0) def end_output(self, **kwargs): """Write end of checking info as csv comment.""" if self.has_part("outro"): self.write_outro() self.close_fileoutput()
class CallbackTester(unittest.TestCase): """ General callback tester to use with specific callback tests. Child class should implement callback_setup() to fill _callback_class, and _fixture attributes. """ def setUp(self): self._out = BytesIO() self.callback_setup() def callback_setup(self): self._callback_class = None self._fixture = {'escape_db_file': 'non_ascii_values.rdb'} def escape_test_helper(self, escape_name): if self._callback_class is None: return # Handle unittest discovery attempt to test with this "abstract" class. escape = getattr(encodehelpers, escape_name) callback = self._callback_class(out=self._out, string_escape=escape) parser = RdbParser(callback) parser.parse( os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR, self._fixture['escape_db_file'])) result = self._out.getvalue() # print('\n%s escape method %s' % (self._callback_class.__name__, escape_name)) # print("\t\tself._fixture['escape_out_%s'] = %s" % (escape, repr(result))) # try: # print(result.decode('utf8')) # except UnicodeDecodeError: # print(result.decode('latin-1')) self.assertEqual(result, self._fixture['escape_out_' + escape], msg='\n%s escape method %s' % (self._callback_class.__name__, escape_name)) def test_raw_escape(self): """Test using STRING_ESCAPE_RAW with varied key encodings against expected output.""" self.escape_test_helper('STRING_ESCAPE_RAW') def test_print_escape(self): """Test using STRING_ESCAPE_PRINT with varied key encodings against expected output.""" self.escape_test_helper('STRING_ESCAPE_PRINT') def test_utf8_escape(self): """Test using STRING_ESCAPE_UTF8 with varied key encodings against expected output.""" self.escape_test_helper('STRING_ESCAPE_UTF8') def test_base64_escape(self): """Test using STRING_ESCAPE_BASE64 with varied key encodings against expected output.""" self.escape_test_helper('STRING_ESCAPE_BASE64') def test_all_dumps(self): """Run callback with all test dumps intercepting incidental crashes.""" if self._callback_class is None: return # Handle unittest discovery attempt to test with this "abstract" class. for dump_name in glob.glob( os.path.join(os.path.dirname(__file__), TEST_DUMPS_DIR, '*.rdb')): callback = self._callback_class(out=self._out) parser = RdbParser(callback) try: parser.parse(dump_name) except Exception as err: raise self.failureException( "%s on %s\n%s" % (self._callback_class.__name__, os.path.basename(dump_name), traceback.format_exc())) self._out.seek(0) self._out.truncate()
class CaptureStandardIO(fixtures.Fixture): """Capture stdin, stdout, and stderr. Reading from `sys.stdin` will yield *unicode* strings, much like the default in Python 3. This differs from the usual behaviour in Python 2, so beware. Writing unicode strings to `sys.stdout` or `sys.stderr` will work; they'll be encoded with the `encoding` chosen when creating this fixture. `addInput(...)` should be used to prepare more input to be read. The `output` and `error` properties can be used to obtain what's been written to stdout and stderr. The buffers used internally have the same lifetime as the fixture *instance* itself, so the `output`, and `error` properties remain useful even after the fixture has been cleaned-up, and there's no need to capture them before exiting. However, `clearInput()`, `clearOutput()`, `clearError()`, and `clearAll()` can be used to truncate the buffers during this fixture's lifetime. """ stdin = None stdout = None stderr = None def __init__(self, encoding="utf-8"): super(CaptureStandardIO, self).__init__() self.codec = codecs.lookup(encoding) self.encoding = encoding # Create new buffers. self._buf_in = BytesIO() self._buf_out = BytesIO() self._buf_err = BytesIO() def setUp(self): super(CaptureStandardIO, self).setUp() self.patcher = MonkeyPatcher() self.addCleanup(self.patcher.restore) # Patch sys.std* and self.std*. Use TextIOWrapper to provide an # identical API to the "real" stdin, stdout, and stderr objects. self._addStream("stdin", self._wrapStream(self._buf_in)) self._addStream("stdout", self._wrapStream(self._buf_out)) self._addStream("stderr", self._wrapStream(self._buf_err)) self.patcher.patch() def _wrapStream(self, stream): return TextIOWrapper(stream, encoding=self.encoding, write_through=True) def _addStream(self, name, stream): self.patcher.add_patch(self, name, stream) self.patcher.add_patch(sys, name, stream) def addInput(self, data): """Add input to be read later, as a unicode string.""" position = self._buf_in.tell() stream = self.codec.streamwriter(self._buf_in) try: self._buf_in.seek(0, 2) stream.write(data) finally: self._buf_in.seek(position) def getInput(self): """The input remaining to be read, as a unicode string.""" position = self._buf_in.tell() if self.stdin is None: stream = self.codec.streamreader(self._buf_in) else: stream = self.stdin try: return stream.read() finally: self._buf_in.seek(position) def getOutput(self): """The output written thus far, as a unicode string.""" if self.stdout is not None: self.stdout.flush() output_bytes = self._buf_out.getvalue() output_string, _ = self.codec.decode(output_bytes) return output_string def getError(self): """The error written thus far, as a unicode string.""" if self.stderr is not None: self.stderr.flush() error_bytes = self._buf_err.getvalue() error_string, _ = self.codec.decode(error_bytes) return error_string def clearInput(self): """Truncate the input buffer.""" self._buf_in.seek(0, 0) self._buf_in.truncate() if self.stdin is not None: self.stdin.seek(0, 0) def clearOutput(self): """Truncate the output buffer.""" self._buf_out.seek(0, 0) self._buf_out.truncate() if self.stdout is not None: self.stdout.seek(0, 0) def clearError(self): """Truncate the error buffer.""" self._buf_err.seek(0, 0) self._buf_err.truncate() if self.stderr is not None: self.stderr.seek(0, 0) def clearAll(self): """Truncate all buffers.""" self.clearInput() self.clearOutput() self.clearError()
class Particle: def __init__(self, jpc_data, particle_offset): self.magic = read_str(jpc_data, particle_offset, 8) assert self.magic == "JEFFjpa1" self.unknown_1 = read_u32(jpc_data, particle_offset + 8) self.num_chunks = read_u32(jpc_data, particle_offset + 0xC) self.size = read_u32(jpc_data, particle_offset + 0x10) # Not accurate in some rare cases self.num_kfa1_chunks = read_u8(jpc_data, particle_offset + 0x14) self.num_fld1_chunks = read_u8(jpc_data, particle_offset + 0x15) self.num_textures = read_u8(jpc_data, particle_offset + 0x16) self.unknown_5 = read_u8(jpc_data, particle_offset + 0x17) self.particle_id = read_u16(jpc_data, particle_offset + 0x18) self.unknown_6 = read_bytes(jpc_data, particle_offset + 0x1A, 6) self.chunks = [] self.chunk_by_type = {} chunk_offset = particle_offset + 0x20 for chunk_index in range(0, self.num_chunks): chunk_magic = read_str(jpc_data, chunk_offset, 4) if chunk_magic in IMPLEMENTED_CHUNK_TYPES: chunk_class = globals().get(chunk_magic, None) else: chunk_class = J3DChunk chunk = chunk_class() chunk.read(jpc_data, chunk_offset) self.chunks.append(chunk) self.chunk_by_type[chunk.magic] = chunk if chunk.magic in IMPLEMENTED_CHUNK_TYPES: setattr(self, chunk.magic.lower(), chunk) chunk_offset += chunk.size self.tdb1.read_texture_ids(self.num_textures) true_size = (chunk_offset - particle_offset) jpc_data.seek(particle_offset) self.data = BytesIO(jpc_data.read(true_size)) def save_changes(self): # Cut off the chunk data first since we're replacing this data entirely. self.data.truncate(0x20) self.data.seek(0x20) self.num_textures = len(self.tdb1.texture_ids) for chunk in self.chunks: chunk.save_changes() chunk.data.seek(0) chunk_data = chunk.data.read() self.data.write(chunk_data) # We don't recalculate this size field, since this is inaccurate anyway. It's probably not even used. #self.size = data_len(self.data) write_magic_str(self.data, 0, self.magic, 8) write_u32(self.data, 0x10, self.size)
def decrypt_aes(file_or_data, password=None, outfile=None, salt=None, mode=None, base64encode=False, chunksize=512 * 1024): """ Flexible implementaiton of AES decryption Parameters ---------- file_or_data : {BufferObject, string, bytes} input data will be converted to bytes sequence for encryption password : {str, None} if None, a prompt will ask for inputing password outfile : {None, path, file} if None, return raw encrypted data salt : {None, string, bytes} salt for password Hashing mode : Cipher.AES.MODE_* chunksize : int encryption chunk, multiple of 16. """ try: from Crypto.Cipher import AES except ImportError as e: raise ImportError("Require 'pycrypto' to run this function") if mode is None: mode = AES.MODE_CBC if password is None: password = input("Your password: "******"Password length must be greater than 0" password = to_password(password, salt) # ====== read header ====== # infile, filesize, own_file = _data_to_iobuffer(file_or_data) origsize = struct.unpack('<Q', infile.read(struct.calcsize('Q')))[0] iv = infile.read(16) decryptor = AES.new(password, mode=AES.MODE_CBC, IV=iv) # ====== outfile ====== # close_file = False if isinstance(outfile, string_types) and os.path.exists( os.path.dirname(outfile)): outfile = open(str(outfile), 'wb') close_file = True elif hasattr(outfile, 'write') and hasattr(outfile, 'flush'): close_file = True else: outfile = BytesIO() # ====== decryption ====== # while True: chunk = infile.read(chunksize) if len(chunk) == 0: break chunk = decryptor.decrypt(chunk) if bool(base64encode): chunk = base64.decodebytes(chunk) outfile.write(chunk) outfile.truncate(origsize) # ====== clean and return ====== # if own_file: infile.close() outfile.flush() if close_file: outfile.close() else: outfile.seek(0) data = outfile.read() outfile.close() return data
class _ResponseReader(protocol.Protocol): def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. if self._finished.called: return self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) if self._maxsize and self._bytes_received > self._maxsize: logger.error( "Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", { 'bytes': self._bytes_received, 'maxsize': self._maxsize, 'request': self._request }) # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: self._reached_warnsize = True logger.warning( "Received more bytes than download " "warn size (%(warnsize)s) in request %(request)s.", { 'warnsize': self._warnsize, 'request': self._request }) def connectionLost(self, reason): if self._finished.called: return body = self._bodybuf.getvalue() if reason.check(ResponseDone): self._finished.callback((self._txresponse, body, None)) return if reason.check(PotentialDataLoss): self._finished.callback((self._txresponse, body, ['partial'])) return if reason.check(ResponseFailed) and any( r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: self._finished.callback((self._txresponse, body, ['dataloss'])) return elif not self._fail_on_dataloss_warned: logger.warning( "Got data loss in %s. If you want to process broken " "responses set the setting DOWNLOAD_FAIL_ON_DATALOSS = False" " -- This message won't be shown in further requests", self._txresponse.request.absoluteURI.decode()) self._fail_on_dataloss_warned = True self._finished.errback(reason)
def captureGenerator(self): stream = BytesIO() for foo in range(self.shutter_auto_wait): yield stream stream.seek(0) stream.truncate(0) if self.capture_method == CAPTURE_BASIC: pass elif self.capture_method == CAPTURE_ON_FRAME: motor.direction = MOTOR_FORWARD elif self.capture_method == CAPTURE_ON_TRIGGER: if self.bracket_steps != 1: #Reduce motor speed frames = 3 * self.shutter_auto_wait + self.shutter_speed_wait motor.speed = float(self.framerate) / frames msg = "Reducing motor speed to %f" % (motor.speed) headermsg = {'type': HEADER_MESSAGE, 'msg': msg} queue.put(headermsg) print( 'Capture on trigger with bracket reducing motor speed to', motor.speed) motor.direction = MOTOR_FORWARD motor.advance() header = {'type': HEADER_IMAGE} while captureEvent.isSet(): if not restartEvent.isSet(): msgheader = {'type': HEADER_MESSAGE, 'msg': 'Pausing capture'} queue.put(msgheader) restartEvent.wait() msgheader = {'type': HEADER_MESSAGE, 'msg': 'Resuming capture'} queue.put(msgheader) if self.capture_method == CAPTURE_ON_TRIGGER: triggerEvent.wait() elif self.capture_method == CAPTURE_ON_FRAME: motor.advanceUntilTrigger() #bypass old frames for foo in range(self.shutter_auto_wait): yield stream stream.seek(0) stream.truncate(0) #Wait if queue is full if queue.qsize() > 20: if self.capture_method == CAPTURE_ON_TRIGGER: motor.stop() msgheader = {'type': HEADER_MESSAGE, 'msg': 'Capture paused'} queue.put(msgheader) print('Warning queue > 20\n', flush=True) while queue.qsize() > 1: time.sleep(1) msgheader = {'type': HEADER_MESSAGE, 'msg': 'Capture resumed'} queue.put(msgheader) if self.capture_method == CAPTURE_ON_TRIGGER: motor.advance() self.frameCounter = self.frameCounter + 1 #No bracket if self.bracket_steps == 1: yield stream stream.seek(0) self.putHeader(0) queue.put(stream.getvalue()) stream.truncate(0) else: #Do braket #First shot image #3 Normal (auto) #Second shot image #2 light auto*light coeff #Third shot image #1 dark auto*dark coeff ## coef = (self.bracket_light_coefficient, self.bracket_dark_coefficient,0) #normal clair sombre ## autoExposureSpeed = self.exposure_speed #First shoot ## for i in range(self.bracket_steps) : ## exposureSpeed = int(autoExposureSpeed * coef[i]) #Exposure for next shot last is 0 (auto) ## self.shutter_speed = exposureSpeed ## yield stream ## stream.seek(0) ## self.putHeader(self.bracket_steps - i) #First is 3 Last is 1 ## queue.put(stream.getvalue()) ## stream.truncate(0) ## for foo in range(self.shutter_speed_wait) : ## yield stream ## stream.seek(0) ## stream.truncate(0) #Normal autoExposureSpeed = self.exposure_speed #First shoot self.shutter_speed = int(autoExposureSpeed * self.bracket_light_coefficient) yield stream stream.seek(0) self.putHeader(3) #First is 3 Last is 1 queue.put(stream.getvalue()) stream.truncate(0) #Overexposed for foo in range(self.shutter_speed_wait): yield stream stream.seek(0) stream.truncate(0) self.exposure_mode = 'off' #lock the gains self.shutter_speed = int(autoExposureSpeed * self.bracket_dark_coefficient) yield stream stream.seek(0) self.putHeader(2) #First is 3 Last is 1 queue.put(stream.getvalue()) stream.truncate(0) #Underexposed for foo in range(self.shutter_speed_wait): yield stream stream.seek(0) stream.truncate(0) self.shutter_speed = autoExposureSpeed self.exposure_mode = 'auto' yield stream stream.seek(0) self.putHeader(1) #First is 3 Last is 1 queue.put(stream.getvalue()) stream.truncate(0) for foo in range(self.shutter_speed_wait): yield stream stream.seek(0) stream.truncate(0) self.shutter_speed = 0 if self.capture_method == CAPTURE_ON_TRIGGER: motor.stop()
def __call__(self): self.control_bytes = self.calculate_control_bytes_for_each_entry( self.entries) index_blocks, idxt_blocks, record_counts, last_indices = [BytesIO()], [ BytesIO() ], [0], [b''] buf = BytesIO() RECORD_LIMIT = 0x10000 - self.HEADER_LENGTH - 1048 # kindlegen uses 1048 (there has to be some margin because of block alignment) for i, (index_num, tags) in enumerate(self.entries): control_bytes = self.control_bytes[i] buf.seek(0), buf.truncate(0) index_num = (index_num.encode('utf-8') if isinstance( index_num, unicode_type) else index_num) raw = bytearray(index_num) raw.insert(0, len(index_num)) buf.write(bytes(raw)) buf.write(bytes(bytearray(control_bytes))) for tag in self.tag_types: values = tags.get(tag.name, None) if values is None: continue try: len(values) except TypeError: values = [values] if values: for val in values: try: buf.write(encint(val)) except ValueError: raise ValueError('Invalid values for %r: %r' % (tag, values)) raw = buf.getvalue() offset = index_blocks[-1].tell() idxt_pos = idxt_blocks[-1].tell() if offset + idxt_pos + len(raw) + 2 > RECORD_LIMIT: index_blocks.append(BytesIO()) idxt_blocks.append(BytesIO()) record_counts.append(0) offset = idxt_pos = 0 last_indices.append(b'') record_counts[-1] += 1 idxt_blocks[-1].write(pack(b'>H', self.HEADER_LENGTH + offset)) index_blocks[-1].write(raw) last_indices[-1] = index_num index_records = [] for index_block, idxt_block, record_count in zip( index_blocks, idxt_blocks, record_counts): index_block = align_block(index_block.getvalue()) idxt_block = align_block(b'IDXT' + idxt_block.getvalue()) # Create header for this index record header = b'INDX' buf.seek(0), buf.truncate(0) buf.write(pack(b'>I', self.HEADER_LENGTH)) buf.write(b'\0' * 4) # Unknown buf.write( pack(b'>I', 1) ) # Header type (0 for Index header record and 1 for Index records) buf.write(b'\0' * 4) # Unknown # IDXT block offset buf.write(pack(b'>I', self.HEADER_LENGTH + len(index_block))) # Number of index entries in this record buf.write(pack(b'>I', record_count)) buf.write(b'\xff' * 8) # Unknown buf.write(b'\0' * 156) # Unknown header += buf.getvalue() index_records.append(header + index_block + idxt_block) if len(index_records[-1]) > 0x10000: raise ValueError( 'Failed to rollover index blocks for very large index.') # Create the Index Header record tagx = self.generate_tagx() # Geometry of the index records is written as index entries pointed to # by the IDXT records buf.seek(0), buf.truncate() idxt = [b'IDXT'] pos = IndexHeader.HEADER_LENGTH + len(tagx) for last_idx, num in zip(last_indices, record_counts): start = buf.tell() idxt.append(pack(b'>H', pos)) buf.write(bytes(bytearray([len(last_idx)])) + last_idx) buf.write(pack(b'>H', num)) pos += buf.tell() - start header = { 'num_of_entries': sum(r for r in record_counts), 'num_of_records': len(index_records), 'num_of_cncx': len(self.cncx), 'tagx': align_block(tagx), 'geometry': align_block(buf.getvalue()), 'idxt': align_block(b''.join(idxt)), } header = IndexHeader()(**header) self.records = [header] + index_records self.records.extend(self.cncx.records) return self.records
#img.save(captureTime + ".jpg") # Detect cat cat_present = detect_objects(interpreter, img, 0.15) print("cat present:", cat_present) # Send POST request to webserver payload = {'status': int(cat_present)} myHeaders = {'Content-type': 'application/json'} r = req.post(url, json=payload, headers=myHeaders) if r.status_code == 200: content = r.json() print(captureTime, content) # Cooldown timer if dispenser is activated if content['action'] == 'dispensed': print("Waiting 1 min") time.sleep(60) else: print("wait 0.5 sec") time.sleep(0.5) else: print("server error: ", r) except Exception as e: print("exception:", e.args) finally: # Clear stream and reset position stream.truncate(0) stream.seek(0)
class Curl: "High-level interface to pycurl functions." def __init__(self, base_url="", fakeheaders=None): self.handle = pycurl.Curl() # These members might be set. self.set_url(base_url) self.verbosity = 0 self.fakeheaders = fakeheaders or [] # Nothing past here should be modified by the caller. self.payload = None self.payload_io = BytesIO() self.hdr = "" # Verify that we've got the right site; harmless on a non-SSL connect. self.set_option(pycurl.SSL_VERIFYHOST, 2) # Follow redirects in case it wants to take us to a CGI... self.set_option(pycurl.FOLLOWLOCATION, 1) self.set_option(pycurl.MAXREDIRS, 5) self.set_option(pycurl.NOSIGNAL, 1) # Setting this option with even a nonexistent file makes libcurl # handle cookie capture and playback automatically. self.set_option(pycurl.COOKIEFILE, "/dev/null") # Set timeouts to avoid hanging too long self.set_timeout(30) # Use password identification from .netrc automatically self.set_option(pycurl.NETRC, 1) self.set_option(pycurl.WRITEFUNCTION, self.payload_io.write) def header_callback(x): self.hdr += x.decode('ascii') self.set_option(pycurl.HEADERFUNCTION, header_callback) def set_timeout(self, timeout): "Set timeout for a retrieving an object" self.set_option(pycurl.TIMEOUT, timeout) def set_url(self, url): "Set the base URL to be retrieved." self.base_url = url self.set_option(pycurl.URL, self.base_url) def set_option(self, *args): "Set an option on the retrieval." self.handle.setopt(*args) def set_verbosity(self, level): "Set verbosity to 1 to see transactions." self.set_option(pycurl.VERBOSE, level) def __request(self, relative_url=None): "Perform the pending request." if self.fakeheaders: self.set_option(pycurl.HTTPHEADER, self.fakeheaders) if relative_url: self.set_option(pycurl.URL, urljoin(self.base_url, relative_url)) self.payload = None self.payload_io.seek(0) self.payload_io.truncate() self.hdr = "" self.handle.perform() self.payload = self.payload_io.getvalue() return self.payload def get(self, url="", params=None): "Ship a GET request for a specified URL, capture the response." if params: url += "?" + urllib_parse.urlencode(params) self.set_option(pycurl.HTTPGET, 1) return self.__request(url) def head(self, url="", params=None): "Ship a HEAD request for a specified URL, capture the response." if params: url += "?" + urllib_parse.urlencode(params) self.set_option(pycurl.NOBODY, 1) return self.__request(url) def post(self, cgi, params): "Ship a POST request to a specified CGI, capture the response." self.set_option(pycurl.POST, 1) self.set_option(pycurl.POSTFIELDS, urllib_parse.urlencode(params)) return self.__request(cgi) def body(self): "Return the body from the last response." return self.payload def header(self): "Return the header from the last response." return self.hdr def get_info(self, *args): "Get information about retrieval." return self.handle.getinfo(*args) def info(self): "Return a dictionary with all info on the last response." m = {} m['effective-url'] = self.handle.getinfo(pycurl.EFFECTIVE_URL) m['http-code'] = self.handle.getinfo(pycurl.HTTP_CODE) m['total-time'] = self.handle.getinfo(pycurl.TOTAL_TIME) m['namelookup-time'] = self.handle.getinfo(pycurl.NAMELOOKUP_TIME) m['connect-time'] = self.handle.getinfo(pycurl.CONNECT_TIME) m['pretransfer-time'] = self.handle.getinfo(pycurl.PRETRANSFER_TIME) m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) m['size-upload'] = self.handle.getinfo(pycurl.SIZE_UPLOAD) m['size-download'] = self.handle.getinfo(pycurl.SIZE_DOWNLOAD) m['speed-upload'] = self.handle.getinfo(pycurl.SPEED_UPLOAD) m['header-size'] = self.handle.getinfo(pycurl.HEADER_SIZE) m['request-size'] = self.handle.getinfo(pycurl.REQUEST_SIZE) m['content-length-download'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD) m['content-length-upload'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_UPLOAD) m['content-type'] = self.handle.getinfo(pycurl.CONTENT_TYPE) m['response-code'] = self.handle.getinfo(pycurl.RESPONSE_CODE) m['speed-download'] = self.handle.getinfo(pycurl.SPEED_DOWNLOAD) m['ssl-verifyresult'] = self.handle.getinfo(pycurl.SSL_VERIFYRESULT) m['filetime'] = self.handle.getinfo(pycurl.INFO_FILETIME) m['starttransfer-time'] = self.handle.getinfo(pycurl.STARTTRANSFER_TIME) m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME) m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT) m['http-connectcode'] = self.handle.getinfo(pycurl.HTTP_CONNECTCODE) m['httpauth-avail'] = self.handle.getinfo(pycurl.HTTPAUTH_AVAIL) m['proxyauth-avail'] = self.handle.getinfo(pycurl.PROXYAUTH_AVAIL) m['os-errno'] = self.handle.getinfo(pycurl.OS_ERRNO) m['num-connects'] = self.handle.getinfo(pycurl.NUM_CONNECTS) m['ssl-engines'] = self.handle.getinfo(pycurl.SSL_ENGINES) m['cookielist'] = self.handle.getinfo(pycurl.INFO_COOKIELIST) m['lastsocket'] = self.handle.getinfo(pycurl.LASTSOCKET) m['ftp-entry-path'] = self.handle.getinfo(pycurl.FTP_ENTRY_PATH) return m def answered(self, check): "Did a given check string occur in the last payload?" return self.payload.find(check) >= 0 def close(self): "Close a session, freeing resources." if self.handle: self.handle.close() self.handle = None self.hdr = "" self.payload = "" def __del__(self): self.close()
class Renderer(object): """Helper class for building DNS wire-format messages. Most applications can use the higher-level L{dns.message.Message} class and its to_wire() method to generate wire-format messages. This class is for those applications which need finer control over the generation of messages. Typical use:: r = dns.renderer.Renderer(id=1, flags=0x80, max_size=512) r.add_question(qname, qtype, qclass) r.add_rrset(dns.renderer.ANSWER, rrset_1) r.add_rrset(dns.renderer.ANSWER, rrset_2) r.add_rrset(dns.renderer.AUTHORITY, ns_rrset) r.add_edns(0, 0, 4096) r.add_rrset(dns.renderer.ADDTIONAL, ad_rrset_1) r.add_rrset(dns.renderer.ADDTIONAL, ad_rrset_2) r.write_header() r.add_tsig(keyname, secret, 300, 1, 0, '', request_mac) wire = r.get_wire() @ivar output: where rendering is written @type output: BytesIO object @ivar id: the message id @type id: int @ivar flags: the message flags @type flags: int @ivar max_size: the maximum size of the message @type max_size: int @ivar origin: the origin to use when rendering relative names @type origin: dns.name.Name object @ivar compress: the compression table @type compress: dict @ivar section: the section currently being rendered @type section: int (dns.renderer.QUESTION, dns.renderer.ANSWER, dns.renderer.AUTHORITY, or dns.renderer.ADDITIONAL) @ivar counts: list of the number of RRs in each section @type counts: int list of length 4 @ivar mac: the MAC of the rendered message (if TSIG was used) @type mac: string """ def __init__(self, id=None, flags=0, max_size=65535, origin=None): """Initialize a new renderer. @param id: the message id @type id: int @param flags: the DNS message flags @type flags: int @param max_size: the maximum message size; the default is 65535. If rendering results in a message greater than I{max_size}, then L{dns.exception.TooBig} will be raised. @type max_size: int @param origin: the origin to use when rendering relative names @type origin: dns.name.Name or None. """ self.output = BytesIO() if id is None: self.id = random.randint(0, 65535) else: self.id = id self.flags = flags self.max_size = max_size self.origin = origin self.compress = {} self.section = QUESTION self.counts = [0, 0, 0, 0] self.output.write(b'\x00' * 12) self.mac = '' def _rollback(self, where): """Truncate the output buffer at offset I{where}, and remove any compression table entries that pointed beyond the truncation point. @param where: the offset @type where: int """ self.output.seek(where) self.output.truncate() keys_to_delete = [] for k, v in self.compress.items(): if v >= where: keys_to_delete.append(k) for k in keys_to_delete: del self.compress[k] def _set_section(self, section): """Set the renderer's current section. Sections must be rendered order: QUESTION, ANSWER, AUTHORITY, ADDITIONAL. Sections may be empty. @param section: the section @type section: int @raises dns.exception.FormError: an attempt was made to set a section value less than the current section. """ if self.section != section: if self.section > section: raise dns.exception.FormError self.section = section def add_question(self, qname, rdtype, rdclass=dns.rdataclass.IN): """Add a question to the message. @param qname: the question name @type qname: dns.name.Name @param rdtype: the question rdata type @type rdtype: int @param rdclass: the question rdata class @type rdclass: int """ self._set_section(QUESTION) before = self.output.tell() qname.to_wire(self.output, self.compress, self.origin) self.output.write(struct.pack("!HH", rdtype, rdclass)) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise dns.exception.TooBig self.counts[QUESTION] += 1 def add_rrset(self, section, rrset, **kw): """Add the rrset to the specified section. Any keyword arguments are passed on to the rdataset's to_wire() routine. @param section: the section @type section: int @param rrset: the rrset @type rrset: dns.rrset.RRset object """ self._set_section(section) before = self.output.tell() n = rrset.to_wire(self.output, self.compress, self.origin, **kw) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise dns.exception.TooBig self.counts[section] += n def add_rdataset(self, section, name, rdataset, **kw): """Add the rdataset to the specified section, using the specified name as the owner name. Any keyword arguments are passed on to the rdataset's to_wire() routine. @param section: the section @type section: int @param name: the owner name @type name: dns.name.Name object @param rdataset: the rdataset @type rdataset: dns.rdataset.Rdataset object """ self._set_section(section) before = self.output.tell() n = rdataset.to_wire(name, self.output, self.compress, self.origin, **kw) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise dns.exception.TooBig self.counts[section] += n def add_edns(self, edns, ednsflags, payload, options=None): """Add an EDNS OPT record to the message. @param edns: The EDNS level to use. @type edns: int @param ednsflags: EDNS flag values. @type ednsflags: int @param payload: The EDNS sender's payload field, which is the maximum size of UDP datagram the sender can handle. @type payload: int @param options: The EDNS options list @type options: list of dns.edns.Option instances @see: RFC 2671 """ # make sure the EDNS version in ednsflags agrees with edns ednsflags &= long(0xFF00FFFF) ednsflags |= (edns << 16) self._set_section(ADDITIONAL) before = self.output.tell() self.output.write( struct.pack('!BHHIH', 0, dns.rdatatype.OPT, payload, ednsflags, 0)) if options is not None: lstart = self.output.tell() for opt in options: stuff = struct.pack("!HH", opt.otype, 0) self.output.write(stuff) start = self.output.tell() opt.to_wire(self.output) end = self.output.tell() assert end - start < 65536 self.output.seek(start - 2) stuff = struct.pack("!H", end - start) self.output.write(stuff) self.output.seek(0, 2) lend = self.output.tell() assert lend - lstart < 65536 self.output.seek(lstart - 2) stuff = struct.pack("!H", lend - lstart) self.output.write(stuff) self.output.seek(0, 2) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise dns.exception.TooBig self.counts[ADDITIONAL] += 1 def add_tsig(self, keyname, secret, fudge, id, tsig_error, other_data, request_mac, algorithm=dns.tsig.default_algorithm): """Add a TSIG signature to the message. @param keyname: the TSIG key name @type keyname: dns.name.Name object @param secret: the secret to use @type secret: string @param fudge: TSIG time fudge @type fudge: int @param id: the message id to encode in the tsig signature @type id: int @param tsig_error: TSIG error code; default is 0. @type tsig_error: int @param other_data: TSIG other data. @type other_data: string @param request_mac: This message is a response to the request which had the specified MAC. @type request_mac: string @param algorithm: the TSIG algorithm to use @type algorithm: dns.name.Name object """ self._set_section(ADDITIONAL) before = self.output.tell() s = self.output.getvalue() (tsig_rdata, self.mac, ctx) = dns.tsig.sign(s, keyname, secret, int(time.time()), fudge, id, tsig_error, other_data, request_mac, algorithm=algorithm) keyname.to_wire(self.output, self.compress, self.origin) self.output.write( struct.pack('!HHIH', dns.rdatatype.TSIG, dns.rdataclass.ANY, 0, 0)) rdata_start = self.output.tell() self.output.write(tsig_rdata) after = self.output.tell() assert after - rdata_start < 65536 if after >= self.max_size: self._rollback(before) raise dns.exception.TooBig self.output.seek(rdata_start - 2) self.output.write(struct.pack('!H', after - rdata_start)) self.counts[ADDITIONAL] += 1 self.output.seek(10) self.output.write(struct.pack('!H', self.counts[ADDITIONAL])) self.output.seek(0, 2) def write_header(self): """Write the DNS message header. Writing the DNS message header is done after all sections have been rendered, but before the optional TSIG signature is added. """ self.output.seek(0) self.output.write( struct.pack('!HHHHHH', self.id, self.flags, self.counts[0], self.counts[1], self.counts[2], self.counts[3])) self.output.seek(0, 2) def get_wire(self): """Return the wire format message. @rtype: string """ return self.output.getvalue()