def test_simple(self): c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1], [3, 1, 2]]) self.repo.refs["HEAD"] = c3.id outstream = BytesIO() porcelain.show(self.repo.path, objects=[c3.id], outstream=outstream) self.assertTrue(outstream.getvalue().startswith("-" * 50))
class PreambleTestCase(unittest.TestCase): class doc_info: doc_id = 'D-deadbeef' rev = '397932e0c77f45fcb7c3732930e7e9b2:1' def setUp(self): self.cleartext = BytesIO(snowden1) self.blob = _crypto.BlobEncryptor( self.doc_info, self.cleartext, secret='A' * 96) def test_preamble_starts_with_magic_signature(self): preamble = self.blob._encode_preamble() assert preamble.startswith(_crypto.MAGIC) def test_preamble_has_cipher_metadata(self): preamble = self.blob._encode_preamble() unpacked = _preamble.PACMAN.unpack(preamble) encryption_scheme, encryption_method = unpacked[1:3] assert encryption_scheme in _crypto.ENC_SCHEME assert encryption_method in _crypto.ENC_METHOD assert unpacked[4] == self.blob.iv def test_preamble_has_document_sync_metadata(self): preamble = self.blob._encode_preamble() unpacked = _preamble.PACMAN.unpack(preamble) doc_id, doc_rev = unpacked[5:7] assert doc_id == self.doc_info.doc_id assert doc_rev == self.doc_info.rev def test_preamble_has_document_size(self): preamble = self.blob._encode_preamble() unpacked = _preamble.PACMAN.unpack(preamble) size = unpacked[7] assert size == _crypto._ceiling(len(snowden1)) @defer.inlineCallbacks def test_preamble_can_come_without_size(self): # XXX: This test case is here only to test backwards compatibility! preamble = self.blob._encode_preamble() # repack preamble using legacy format, without doc size unpacked = _preamble.PACMAN.unpack(preamble) preamble_without_size = _preamble.LEGACY_PACMAN.pack(*unpacked[0:7]) # encrypt it manually for custom tag ciphertext, tag = _aes_encrypt(self.blob.sym_key, self.blob.iv, self.cleartext.getvalue(), aead=preamble_without_size) ciphertext = ciphertext + tag # encode it ciphertext = base64.urlsafe_b64encode(ciphertext) preamble_without_size = base64.urlsafe_b64encode(preamble_without_size) # decrypt it ciphertext = preamble_without_size + ' ' + ciphertext cleartext = yield _crypto.BlobDecryptor( self.doc_info, BytesIO(ciphertext), secret='A' * 96).decrypt() assert cleartext.getvalue() == self.cleartext.getvalue() warnings = self.flushWarnings() assert len(warnings) == 1 assert 'legacy preamble without size' in warnings[0]['message']
def test_store(self): out = BytesIO() storage = StdoutFeedStorage('stdout:', _stdout=out) file = storage.open(scrapy.Spider("default")) file.write(b"content") yield storage.store(file) self.assertEqual(out.getvalue(), b"content")
def test_empty(self): c1, c2, c3 = build_commit_graph(self.repo.object_store, [[1], [2, 1], [3, 1, 2]]) self.repo.refs[b"HEAD"] = c3.id outstream = BytesIO() porcelain.diff_tree(self.repo.path, c2.tree, c3.tree, outstream=outstream) self.assertEqual(outstream.getvalue(), b"")
def test_read_atom(self): """Test that all fields are read from PDB ATOM records""" s = BytesIO() # PDB is fixed-format; we should be able to read coordinates even # without spaces between them s.write(b'ATOM 1 N ALA A 5 3000.0001000.4002000.600' b' 2.00 6.40 N\n') s.seek(0) m = IMP.Model() pdb = IMP.atom.read_pdb(s, m) atoms = IMP.atom.get_by_type(pdb, IMP.atom.ATOM_TYPE) self.assertEqual(len(atoms), 1) a = IMP.atom.Atom(atoms[0]) r = IMP.atom.Residue(a.get_parent()) c = IMP.atom.Chain(r.get_parent()) self.assertEqual(a.get_input_index(), 1) self.assertEqual(a.get_atom_type().get_string(), 'N') # Note: currently don't read alternate location or insertion code self.assertEqual(r.get_residue_type().get_string(), 'ALA') self.assertEqual(c.get_id(), 'A') self.assertEqual(r.get_index(), 5) coord = IMP.core.XYZ(a).get_coordinates() self.assertAlmostEqual(coord[0], 3000.000, delta=0.001) self.assertAlmostEqual(coord[1], 1000.400, delta=0.001) self.assertAlmostEqual(coord[2], 2000.600, delta=0.001) self.assertAlmostEqual(a.get_occupancy(), 2.00, delta=0.01) self.assertAlmostEqual(a.get_temperature_factor(), 6.40, delta=0.01)
def test_save_dict(): # Test that dict can be saved (as recarray), loaded as matstruct dict_types = ((dict, False),) try: from collections import OrderedDict except ImportError: pass else: dict_types += ((OrderedDict, True),) ab_exp = np.array([[(1, 2)]], dtype=[('a', object), ('b', object)]) ba_exp = np.array([[(2, 1)]], dtype=[('b', object), ('a', object)]) for dict_type, is_ordered in dict_types: # Initialize with tuples to keep order for OrderedDict d = dict_type([('a', 1), ('b', 2)]) stream = BytesIO() savemat(stream, {'dict': d}) stream.seek(0) vals = loadmat(stream)['dict'] assert_equal(set(vals.dtype.names), set(['a', 'b'])) if is_ordered: # Input was ordered, output in ab order assert_array_equal(vals, ab_exp) else: # Not ordered input, either order output if vals.dtype.names[0] == 'a': assert_array_equal(vals, ab_exp) else: assert_array_equal(vals, ba_exp)
def test_read_opts(): # tests if read is seeing option sets, at initialization and after # initialization arr = np.arange(6).reshape(1,6) stream = BytesIO() savemat(stream, {'a': arr}) rdr = MatFile5Reader(stream) back_dict = rdr.get_variables() rarr = back_dict['a'] assert_array_equal(rarr, arr) rdr = MatFile5Reader(stream, squeeze_me=True) assert_array_equal(rdr.get_variables()['a'], arr.reshape((6,))) rdr.squeeze_me = False assert_array_equal(rarr, arr) rdr = MatFile5Reader(stream, byte_order=boc.native_code) assert_array_equal(rdr.get_variables()['a'], arr) # inverted byte code leads to error on read because of swapped # header etc rdr = MatFile5Reader(stream, byte_order=boc.swapped_code) assert_raises(Exception, rdr.get_variables) rdr.byte_order = boc.native_code assert_array_equal(rdr.get_variables()['a'], arr) arr = np.array(['a string']) stream.truncate(0) stream.seek(0) savemat(stream, {'a': arr}) rdr = MatFile5Reader(stream) assert_array_equal(rdr.get_variables()['a'], arr) rdr = MatFile5Reader(stream, chars_as_strings=False) carr = np.atleast_2d(np.array(list(arr.item()), dtype='U1')) assert_array_equal(rdr.get_variables()['a'], carr) rdr.chars_as_strings = True assert_array_equal(rdr.get_variables()['a'], arr)
def __init__(self, buf): self._progress = 0 self._len = len(buf) self._bar = None if self._len > 4096: self._bar = progress.Bar(filled_char='=', every=4096) BytesIO.__init__(self, buf)
def _serialize_properties(self): """serialize the 'properties' attribute (a dictionary) into the raw bytes making up a set of property flags and a property list, suitable for putting into a content frame header.""" shift = 15 flag_bits = 0 flags = [] sformat, svalues = [], [] props = self.properties props.setdefault('content_encoding', 'utf-8') for key, proptype in self.PROPERTIES: val = props.get(key, None) if val is not None: if shift == 0: flags.append(flag_bits) flag_bits = 0 shift = 15 flag_bits |= (1 << shift) if proptype != 'bit': sformat.append(proptype) svalues.append(val) shift -= 1 flags.append(flag_bits) result = BytesIO() write = result.write for flag_bits in flags: write(pack('>H', flag_bits)) write(dumps(''.join(sformat), svalues)) return result.getvalue()
def get_compressed_file_data(file_path, compresslevel=5): compressed_buffer = BytesIO() gzip_file = GzipFile(mode='wb', compresslevel=compresslevel, fileobj=compressed_buffer) try: fileobj = open(file_path, 'rb') while True: x = fileobj.read(65536) if not x: break gzip_file.write(x) x = None fileobj.close() except IOError as e: LOG.error(str(e)) return None gzip_file.close() compressed_data = compressed_buffer.getvalue() compressed_buffer.close() return compressed_data
def open(self, name, mode='rb'): resp = self.b2.download_file(name) output = BytesIO() output.write(resp) output.seek(0) return File(output, name)
class FakePayload(object): """ A wrapper around BytesIO that restricts what can be read since data from the network can't be seeked and cannot be read outside of its content length. This makes sure that views can't do anything under the test client that wouldn't work in Real Life. """ def __init__(self, content=None): self.__content = BytesIO() self.__len = 0 self.read_started = False if content is not None: self.write(content) def __len__(self): return self.__len def read(self, num_bytes=None): if not self.read_started: self.__content.seek(0) self.read_started = True if num_bytes is None: num_bytes = self.__len or 0 assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data." content = self.__content.read(num_bytes) self.__len -= num_bytes return content def write(self, content): if self.read_started: raise ValueError("Unable to write a payload after he's been read") content = force_bytes(content) self.__content.write(content) self.__len += len(content)
class TestFileUploadParser(TestCase): def setUp(self): class MockRequest(object): pass from io import BytesIO self.stream = BytesIO( "Test text file".encode('utf-8') ) request = MockRequest() request.upload_handlers = (MemoryFileUploadHandler(),) request.META = { 'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'), 'HTTP_CONTENT_LENGTH': 14, } self.parser_context = {'request': request, 'kwargs': {}} def test_parse(self): """ Make sure the `QueryDict` works OK """ parser = FileUploadParser() self.stream.seek(0) data_and_files = parser.parse(self.stream, None, self.parser_context) file_obj = data_and_files.files['file'] self.assertEqual(file_obj._size, 14) def test_get_filename(self): parser = FileUploadParser() filename = parser.get_filename(self.stream, None, self.parser_context) self.assertEqual(filename, 'file.txt'.encode('utf-8'))
def deserialize(schema, binary): bytes_writer = BytesIO() bytes_writer.write(binary) bytes_writer.seek(0) res = fastavro.schemaless_reader(bytes_writer, schema) return res
def download_sentiments(self): print("Download sentiment data from webservice") f1 = open('../tmp/results.txt', 'w+') for i in range(self.data[:, ].shape[0]): print("Downloading entry " + str(i)) escaped = re.escape(self.data[i, 1]) post_data = {'text': escaped} buffer = BytesIO() c = pycurl.Curl() c.setopt(c.URL, 'http://text-processing.com/api/sentiment/') c.setopt(c.POSTFIELDS, urllib.urlencode(post_data)) c.setopt(c.WRITEDATA, buffer) c.perform() c.close() body = buffer.getvalue() # Body is a byte string. # We have to know the encoding in order to print it to a text file # such as standard output. response = body.decode('utf-8') self.results.append(response) f1.write(str(self.results))
def roundtrip_truncated(arr): f = BytesIO() format.write_array(f, arr) #BytesIO is one byte short f2 = BytesIO(f.getvalue()[0:-1]) arr2 = format.read_array(f2) return arr2
def _save(im, fp, filename): fp.write(_MAGIC) # (2+2) sizes = im.encoderinfo.get("sizes", [(16, 16), (24, 24), (32, 32), (48, 48), (64, 64), (128, 128), (255, 255)]) width, height = im.size filter(lambda x: False if (x[0] > width or x[1] > height or x[0] > 255 or x[1] > 255) else True, sizes) sizes = sorted(sizes, key=lambda x: x[0]) fp.write(struct.pack("H", len(sizes))) # idCount(2) offset = fp.tell() + len(sizes)*16 for size in sizes: width, height = size fp.write(struct.pack("B", width)) # bWidth(1) fp.write(struct.pack("B", height)) # bHeight(1) fp.write(b"\0") # bColorCount(1) fp.write(b"\0") # bReserved(1) fp.write(b"\0\0") # wPlanes(2) fp.write(struct.pack("H", 32)) # wBitCount(2) image_io = BytesIO() tmp = im.copy() tmp.thumbnail(size, Image.ANTIALIAS) tmp.save(image_io, "png") image_io.seek(0) image_bytes = image_io.read() bytes_len = len(image_bytes) fp.write(struct.pack("I", bytes_len)) # dwBytesInRes(4) fp.write(struct.pack("I", offset)) # dwImageOffset(4) current = fp.tell() fp.seek(offset) fp.write(image_bytes) offset = offset + bytes_len fp.seek(current)
class UnicodeWriter: """ A CSV writer which will write rows to CSV file "f", which is encoded in the given encoding. """ def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): # Redirect output to a queue self.queue = BytesIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)('replace') def writerow(self, row): row = [smart_text(s) for s in row] self.writer.writerow([s.encode("utf-8") for s in row]) # Fetch UTF-8 output from the queue ... data = self.queue.getvalue() data = data.decode("utf-8") # ... and reencode it into the target encoding data = self.encoder.encode(data) # write to the target stream self.stream.write(data) # empty queue self.queue.truncate(0) def writerows(self, rows): for row in rows: self.writerow(row)
def to_pptx(self): logger.info('Converting svg -> html -> png -> pptx') content = None try: # convert to png png_fn = self._rasterize_png() # create blank presentation slide layout pres = Presentation() blank_slidelayout = pres.slide_layouts[6] slide = pres.slides.add_slide(blank_slidelayout) self._pptx_add_title(slide) self._pptx_add_url(slide) self._pptx_add_png(slide, png_fn) self._pptx_add_hawc_logo(slide) # save as object content = BytesIO() pres.save(content) content.seek(0) except Exception as e: logger.error(e, exc_info=True) finally: self.cleanup() return content
def test_commit_serialization(self): assert_commit_serialization(self.gitrwrepo, self.gitrwrepo.head, True) rwrepo = self.gitrwrepo make_object = rwrepo.odb.store # direct serialization - deserialization can be tested afterwards # serialization is probably limited on IO hc = rwrepo.commit(rwrepo.head) nc = 5000 st = time() for i in xrange(nc): cm = Commit(rwrepo, Commit.NULL_BIN_SHA, hc.tree, hc.author, hc.authored_date, hc.author_tz_offset, hc.committer, hc.committed_date, hc.committer_tz_offset, str(i), parents=hc.parents, encoding=hc.encoding) stream = BytesIO() cm._serialize(stream) slen = stream.tell() stream.seek(0) cm.binsha = make_object(IStream(Commit.type, slen, stream)).binsha # END commit creation elapsed = time() - st print("Serialized %i commits to loose objects in %f s ( %f commits / s )" % (nc, elapsed, nc / elapsed), file=sys.stderr)
def test_write_struct(): b = BytesIO() item = TItem(id=123, phones=['123456', 'abcdef']) proto.TCyBinaryProtocol(b).write_struct(item) assert_equal("08 00 01 00 00 00 7b 0f 00 02 0b 00 00 00 02 00 00 00 " "06 31 32 33 34 35 36 00 00 00 06 61 62 63 64 65 66 00", hexlify(b.getvalue()))
def test_object_diff_bin_blob_force(self): f = BytesIO() # Prepare two slightly different PNG headers b1 = Blob.from_string( b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52" b"\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x04\x00\x00\x00\x05\x04\x8b" ) b2 = Blob.from_string( b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a\x00\x00\x00\x0d\x49\x48\x44\x52" b"\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x03\x00\x00\x00\x98\xd3\xb3" ) store = MemoryObjectStore() store.add_objects([(b1, None), (b2, None)]) write_object_diff(f, store, (b"foo.png", 0o644, b1.id), (b"bar.png", 0o644, b2.id), diff_binary=True) self.assertEqual( [ b"diff --git a/foo.png b/bar.png", b"index f73e47d..06364b7 644", b"--- a/foo.png", b"+++ b/bar.png", b"@@ -1,4 +1,4 @@", b" \x89PNG", b" \x1a", b" \x00\x00\x00", b"-IHDR\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x04\x00\x00\x00\x05\x04\x8b", b"\\ No newline at end of file", b"+IHDR\x00\x00\x01\xd5\x00\x00\x00\x9f\x08\x03\x00\x00\x00\x98\xd3\xb3", b"\\ No newline at end of file", ], f.getvalue().splitlines(), )
def _process(self): f = request.files['logo'] try: img = Image.open(f) except IOError: flash(_('You cannot upload this file as a logo.'), 'error') return jsonify_data(content=None) if img.format.lower() not in {'jpeg', 'png', 'gif'}: flash(_('The file has an invalid format ({format})').format(format=img.format), 'error') return jsonify_data(content=None) if img.mode == 'CMYK': flash(_('The logo you uploaded is using the CMYK colorspace and has been converted to RGB. Please check if ' 'the colors are correct and convert it manually if necessary.'), 'warning') img = img.convert('RGB') image_bytes = BytesIO() img.save(image_bytes, 'PNG') image_bytes.seek(0) content = image_bytes.read() self.event.logo = content self.event.logo_metadata = { 'hash': crc32(content), 'size': len(content), 'filename': os.path.splitext(secure_filename(f.filename, 'logo'))[0] + '.png', 'content_type': 'image/png' } flash(_('New logo saved'), 'success') logger.info("New logo '%s' uploaded by %s (%s)", f.filename, session.user, self.event) return jsonify_data(content=get_logo_data(self.event))
def process(self, response:Response, responseCnt:ResponseContent, **keyargs): ''' @see: HandlerProcessorProceed.process ''' assert isinstance(response, Response), 'Invalid response %s' % response assert isinstance(responseCnt, ResponseContent), 'Invalid response content %s' % responseCnt if response.isSuccess is False: return # Skip in case the response is in error if Response.encoder not in response: return # Skip in case there is no encoder to render assert callable(response.renderFactory), 'Invalid response renderer factory %s' % response.renderFactory output = BytesIO() render = response.renderFactory(output) assert isinstance(render, IRender), 'Invalid render %s' % render resolve = Resolve(response.encoder).request(value=response.obj, render=render, **response.encoderData or {}) if not self.allowChunked and ResponseContent.length not in responseCnt: while resolve.has(): resolve.do() content = output.getvalue() responseCnt.length = len(content) responseCnt.source = (content,) output.close() else: responseCnt.source = self.renderAsGenerator(resolve, output, self.bufferSize)
def create_dummy_image(): file = BytesIO() image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0)) image.save(file, 'png') file.name = 'test_image.png' file.seek(0) return file
def overwrite_file(node_id, file_name): params = '?suppress=deduplication' # suppresses 409 response buffer = BytesIO() c = pycurl.Curl() c.setopt(c.URL, oauth.get_content_url() + 'nodes/' + node_id + '/content' + params) c.setopt(c.HTTPHEADER, ['Authorization: ' + oauth.get_auth_token()]) c.setopt(c.WRITEDATA, buffer) c.setopt(c.HTTPPOST, [('content', (c.FORM_FILE, file_name.encode('UTF-8')))]) c.setopt(c.CUSTOMREQUEST, 'PUT') c.setopt(c.NOPROGRESS, 0) c.setopt(c.PROGRESSFUNCTION, progress) if logger.getEffectiveLevel() == logging.DEBUG: c.setopt(c.VERBOSE, 1) try: c.perform() except pycurl.error as e: raise RequestError(0, e) status = c.getinfo(pycurl.HTTP_CODE) c.close() print() # break progress line body = buffer.getvalue().decode('utf-8') if status != http.OK: # print('Overwriting "%s" failed.' % file_name) raise RequestError(status, body) return json.loads(body)
def resguardoPdf(request, pk): resguardo = Resguardo.objects.get(id=pk) nombre = 'resguardo_' + str(resguardo.id) response = HttpResponse(content_type='application/pdf') response['Content-Disposition'] = 'attachment;filename=resguardo.pdf' buffer = BytesIO() c = canvas.Canvas(buffer, pagesize=letter) def header(): c.drawImage('static/images/CFPPCH.png', 10, 670, 130, 130) c.setLineWidth(.3) c.setFont('Helvetica-Bold', 20) c.drawString(120, 750, 'CEFPP') c.drawString(160, 740, ) c.setFont('Helvetica-Bold', 15) c.drawString(480, 750, 'Inventario') c.setFillColorRGB(255, 0, 0) c.setFont('Helvetica', 12) c.drawString(485, 735, resguardo.inventario) c.line(460, 747, 560, 747) header() c.showPage() c.save() pdf = buffer.getvalue() buffer.close() response.write(pdf) return response
def test_simple_bytesio(self): f = BytesIO() c = Commit() c.committer = c.author = b"Jelmer <*****@*****.**>" c.commit_time = c.author_time = 1271350201 c.commit_timezone = c.author_timezone = 0 c.message = b"This is the first line\nAnd this is the second line.\n" c.tree = Tree().id write_commit_patch(f, c, b"CONTENTS", (1, 1), version="custom") f.seek(0) lines = f.readlines() self.assertTrue(lines[0].startswith(b"From 0b0d34d1b5b596c928adc9a727a4b9e03d025298")) self.assertEqual(lines[1], b"From: Jelmer <*****@*****.**>\n") self.assertTrue(lines[2].startswith(b"Date: ")) self.assertEqual( [ b"Subject: [PATCH 1/1] This is the first line\n", b"And this is the second line.\n", b"\n", b"\n", b"---\n", ], lines[3:8], ) self.assertEqual([b"CONTENTS-- \n", b"custom\n"], lines[-2:]) if len(lines) >= 12: # diffstat may not be present self.assertEqual(lines[8], b" 0 files changed\n")
def upload_file(file_name, parent=None): params = '?suppress=deduplication' # suppresses 409 response metadata = {'kind': 'FILE', 'name': os.path.basename(file_name)} if parent: metadata['parents'] = [parent] buffer = BytesIO() c = pycurl.Curl() c.setopt(c.URL, oauth.get_content_url() + 'nodes' + params) c.setopt(c.HTTPHEADER, ['Authorization: ' + oauth.get_auth_token()]) c.setopt(c.WRITEDATA, buffer) c.setopt(c.HTTPPOST, [('metadata', json.dumps(metadata)), ('content', (c.FORM_FILE, file_name.encode('UTF-8')))]) c.setopt(c.NOPROGRESS, 0) c.setopt(c.PROGRESSFUNCTION, progress) if logger.getEffectiveLevel() == logging.DEBUG: c.setopt(c.VERBOSE, 1) try: c.perform() except pycurl.error as e: raise RequestError(0, e) status = c.getinfo(pycurl.HTTP_CODE) c.close() print() # break progress line body = buffer.getvalue().decode('utf-8') if status != http.CREATED: # print('Uploading "%s" failed.' % file_name) raise RequestError(status, body) return json.loads(body)
def test_authoritativeMessage(self): """ The L{RRHeader} instances created by L{Message} from an authoritative message are marked as authoritative. """ buf = BytesIO() answer = dns.RRHeader(payload=dns.Record_A('1.2.3.4', ttl=0)) answer.encode(buf) data = ( b'\x01\x00' # Message ID # answer bit, opCode nibble, auth bit, trunc bit, recursive bit b'\x04' # recursion bit, empty bit, empty bit, empty bit, response code # nibble b'\x00' b'\x00\x00' # number of queries b'\x00\x01' # number of answers b'\x00\x00' # number of authorities b'\x00\x00' # number of additionals + buf.getvalue() ) answer.auth = True self.parser.updateData(data) message = self.parser.message() self.assertEqual(message.answers, [answer]) self.assertTrue(message.answers[0].auth)
class Renderer(object): """Helper class for building DNS wire-format messages. Most applications can use the higher-level L{thirdparty.dns.message.Message} class and its to_wire() method to generate wire-format messages. This class is for those applications which need finer control over the generation of messages. Typical use:: r = thirdparty.dns.renderer.Renderer(id=1, flags=0x80, max_size=512) r.add_question(qname, qtype, qclass) r.add_rrset(thirdparty.dns.renderer.ANSWER, rrset_1) r.add_rrset(thirdparty.dns.renderer.ANSWER, rrset_2) r.add_rrset(thirdparty.dns.renderer.AUTHORITY, ns_rrset) r.add_edns(0, 0, 4096) r.add_rrset(thirdparty.dns.renderer.ADDTIONAL, ad_rrset_1) r.add_rrset(thirdparty.dns.renderer.ADDTIONAL, ad_rrset_2) r.write_header() r.add_tsig(keyname, secret, 300, 1, 0, '', request_mac) wire = r.get_wire() output, a BytesIO, where rendering is written id: the message id flags: the message flags max_size: the maximum size of the message origin: the origin to use when rendering relative names compress: the compression table section: an int, the section currently being rendered counts: list of the number of RRs in each section mac: the MAC of the rendered message (if TSIG was used) """ def __init__(self, id=None, flags=0, max_size=65535, origin=None): """Initialize a new renderer.""" self.output = BytesIO() if id is None: self.id = random.randint(0, 65535) else: self.id = id self.flags = flags self.max_size = max_size self.origin = origin self.compress = {} self.section = QUESTION self.counts = [0, 0, 0, 0] self.output.write(b'\x00' * 12) self.mac = '' def _rollback(self, where): """Truncate the output buffer at offset *where*, and remove any compression table entries that pointed beyond the truncation point. """ self.output.seek(where) self.output.truncate() keys_to_delete = [] for k, v in self.compress.items(): if v >= where: keys_to_delete.append(k) for k in keys_to_delete: del self.compress[k] def _set_section(self, section): """Set the renderer's current section. Sections must be rendered order: QUESTION, ANSWER, AUTHORITY, ADDITIONAL. Sections may be empty. Raises thirdparty.dns.exception.FormError if an attempt was made to set a section value less than the current section. """ if self.section != section: if self.section > section: raise thirdparty.dns.exception.FormError self.section = section def add_question(self, qname, rdtype, rdclass=thirdparty.dns.rdataclass.IN): """Add a question to the message.""" self._set_section(QUESTION) before = self.output.tell() qname.to_wire(self.output, self.compress, self.origin) self.output.write(struct.pack("!HH", rdtype, rdclass)) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise thirdparty.dns.exception.TooBig self.counts[QUESTION] += 1 def add_rrset(self, section, rrset, **kw): """Add the rrset to the specified section. Any keyword arguments are passed on to the rdataset's to_wire() routine. """ self._set_section(section) before = self.output.tell() n = rrset.to_wire(self.output, self.compress, self.origin, **kw) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise thirdparty.dns.exception.TooBig self.counts[section] += n def add_rdataset(self, section, name, rdataset, **kw): """Add the rdataset to the specified section, using the specified name as the owner name. Any keyword arguments are passed on to the rdataset's to_wire() routine. """ self._set_section(section) before = self.output.tell() n = rdataset.to_wire(name, self.output, self.compress, self.origin, **kw) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise thirdparty.dns.exception.TooBig self.counts[section] += n def add_edns(self, edns, ednsflags, payload, options=None): """Add an EDNS OPT record to the message.""" # make sure the EDNS version in ednsflags agrees with edns ednsflags &= 0xFF00FFFF ednsflags |= (edns << 16) self._set_section(ADDITIONAL) before = self.output.tell() self.output.write(struct.pack('!BHHIH', 0, thirdparty.dns.rdatatype.OPT, payload, ednsflags, 0)) if options is not None: lstart = self.output.tell() for opt in options: stuff = struct.pack("!HH", opt.otype, 0) self.output.write(stuff) start = self.output.tell() opt.to_wire(self.output) end = self.output.tell() assert end - start < 65536 self.output.seek(start - 2) stuff = struct.pack("!H", end - start) self.output.write(stuff) self.output.seek(0, 2) lend = self.output.tell() assert lend - lstart < 65536 self.output.seek(lstart - 2) stuff = struct.pack("!H", lend - lstart) self.output.write(stuff) self.output.seek(0, 2) after = self.output.tell() if after >= self.max_size: self._rollback(before) raise thirdparty.dns.exception.TooBig self.counts[ADDITIONAL] += 1 def add_tsig(self, keyname, secret, fudge, id, tsig_error, other_data, request_mac, algorithm=thirdparty.dns.tsig.default_algorithm): """Add a TSIG signature to the message.""" s = self.output.getvalue() (tsig_rdata, self.mac, ctx) = thirdparty.dns.tsig.sign(s, keyname, secret, int(time.time()), fudge, id, tsig_error, other_data, request_mac, algorithm=algorithm) self._write_tsig(tsig_rdata, keyname) def add_multi_tsig(self, ctx, keyname, secret, fudge, id, tsig_error, other_data, request_mac, algorithm=thirdparty.dns.tsig.default_algorithm): """Add a TSIG signature to the message. Unlike add_tsig(), this can be used for a series of consecutive DNS envelopes, e.g. for a zone transfer over TCP [RFC2845, 4.4]. For the first message in the sequence, give ctx=None. For each subsequent message, give the ctx that was returned from the add_multi_tsig() call for the previous message.""" s = self.output.getvalue() (tsig_rdata, self.mac, ctx) = thirdparty.dns.tsig.sign(s, keyname, secret, int(time.time()), fudge, id, tsig_error, other_data, request_mac, ctx=ctx, first=ctx is None, multi=True, algorithm=algorithm) self._write_tsig(tsig_rdata, keyname) return ctx def _write_tsig(self, tsig_rdata, keyname): self._set_section(ADDITIONAL) before = self.output.tell() keyname.to_wire(self.output, self.compress, self.origin) self.output.write(struct.pack('!HHIH', thirdparty.dns.rdatatype.TSIG, thirdparty.dns.rdataclass.ANY, 0, 0)) rdata_start = self.output.tell() self.output.write(tsig_rdata) after = self.output.tell() assert after - rdata_start < 65536 if after >= self.max_size: self._rollback(before) raise thirdparty.dns.exception.TooBig self.output.seek(rdata_start - 2) self.output.write(struct.pack('!H', after - rdata_start)) self.counts[ADDITIONAL] += 1 self.output.seek(10) self.output.write(struct.pack('!H', self.counts[ADDITIONAL])) self.output.seek(0, 2) def write_header(self): """Write the DNS message header. Writing the DNS message header is done after all sections have been rendered, but before the optional TSIG signature is added. """ self.output.seek(0) self.output.write(struct.pack('!HHHHHH', self.id, self.flags, self.counts[0], self.counts[1], self.counts[2], self.counts[3])) self.output.seek(0, 2) def get_wire(self): """Return the wire format message.""" return self.output.getvalue()
def main(year, etn): """Go Main Go""" pcursor = POSTGIS.cursor(cursor_factory=psycopg2.extras.DictCursor) basefn = "watch_%s_%s" % (year, etn) os.chdir("/tmp/") sql = """select ST_astext(ST_multi(ST_union(ST_SnapToGrid(u.geom,0.0001)))) as tgeom from warnings_%s w JOIN ugcs u on (u.gid = w.gid) WHERE significance = 'A' and phenomena IN ('TO','SV') and eventid = %s and ST_isvalid(u.geom) and issue < ((select issued from watches WHERE num = %s and extract(year from issued) = %s LIMIT 1) + '60 minutes'::interval) """ % (year, etn, etn, year) pcursor.execute(sql) if pcursor.rowcount == 0: sys.exit() shpio = BytesIO() shxio = BytesIO() dbfio = BytesIO() with shapefile.Writer(shx=shxio, shp=shpio, dbf=dbfio) as shp: shp.field('SIG', 'C', '1') shp.field('ETN', 'I', '4') row = pcursor.fetchone() s = row["tgeom"] f = wellknowntext.convert_well_known_text(s) shp.poly(f) shp.record('A', etn) zio = BytesIO() zf = zipfile.ZipFile(zio, mode='w', compression=zipfile.ZIP_DEFLATED) zf.writestr(basefn+'.prj', open(('/opt/iem/data/gis/meta/4326.prj' )).read()) zf.writestr(basefn+".shp", shpio.getvalue()) zf.writestr(basefn+'.shx', shxio.getvalue()) zf.writestr(basefn+'.dbf', dbfio.getvalue()) zf.close() ssw(("Content-Disposition: attachment; filename=%s.zip\n\n") % (basefn,)) ssw(zio.getvalue())
def compress(inpil): pil = Image.open(inpil) pil.thumbnail((size, size), Image.LANCZOS) pil.save(outpil := BytesIO(), "PNG", optimize=True) return outpil
def serialize(schema, *records): buffer = BytesIO() fastavro.writer(buffer, schema, records) serialized = buffer.getvalue() return serialized
def import_data(bot: Bot, update): msg = update.effective_message # type: Optional[Message] chat = update.effective_chat # type: Optional[Chat] user = update.effective_user # type: Optional[User] # TODO: allow uploading doc with command, not just as reply # only work with a doc conn = connected(bot, update, chat, user.id, need_admin=True) if conn: chat = dispatcher.bot.getChat(conn) chat_id = conn chat_name = dispatcher.bot.getChat(conn).title else: if update.effective_message.chat.type == "private": update.effective_message.reply_text( "This command can only be runned on group, not PM.") return "" chat = update.effective_chat chat_id = update.effective_chat.id chat_name = update.effective_message.chat.title if msg.reply_to_message and msg.reply_to_message.document: filetype = msg.reply_to_message.document.file_name if filetype.split('.')[-1] not in ("backup", "json", "txt"): msg.reply_text("File is not valid!") return try: file_info = bot.get_file(msg.reply_to_message.document.file_id) except BadRequest: msg.reply_text( "Try downloading and uploading the file yourself again, This one seem broken!" ) return with BytesIO() as file: file_info.download(out=file) file.seek(0) data = json.load(file) try: # If backup is from Monica if data.get('bot_base') == "Monica": imp_notes = 0 NOT_IMPORTED = "This cannot be imported because from other bot." NOT_IMPORTED_INT = 0 # If backup is from this bot, import all files if data.get('bot_id') == bot.id: is_self = True else: is_self = False # Import notes if data.get('notes'): allnotes = data['notes'] NOT_IMPORTED += "\n\nNotes:\n" for x in allnotes: # If from self, import all if is_self: note_data, buttons = button_markdown_parser( x['note_data'], entities=0) note_name = x['note_tag'] note_file = None note_type = x['note_type'] if x['note_file']: note_file = x['note_file'] if note_type == 0: note_type = Types.TEXT elif note_type == 1: note_type = Types.BUTTON_TEXT elif note_type == 2: note_type = Types.STICKER elif note_type == 3: note_type = Types.DOCUMENT elif note_type == 4: note_type = Types.PHOTO elif note_type == 5: note_type = Types.AUDIO elif note_type == 6: note_type = Types.VOICE elif note_type == 7: note_type = Types.VIDEO elif note_type == 8: note_type = Types.VIDEO_NOTE else: note_type = None if note_type <= 8: notesql.add_note_to_db(chat_id, note_name, note_data, note_type, buttons, note_file) imp_notes += 1 else: # If this text if x['note_type'] == 0: note_data, buttons = button_markdown_parser( x['text'].replace("\\", ""), entities=0) note_name = x['name'] notesql.add_note_to_db(chat_id, note_name, note_data, Types.TEXT, buttons, None) imp_notes += 1 else: NOT_IMPORTED += "- {}\n".format(x['name']) NOT_IMPORTED_INT += 1 if conn: text = (update.effective_message, "Full backup returned on *{}*. Welcome backup! " ).format(chat_name) else: text = ( update.effective_message, "Backup fully restored.\nDone with welcome backup! " ).format(chat_name) try: msg.reply_text(text, parse_mode="markdown") except BadRequest: msg.reply_text(text, parse_mode="markdown", quote=False) if NOT_IMPORTED_INT: f = open("{}-notimported.txt".format(chat_id), "w") f.write(str(NOT_IMPORTED)) f.close() bot.sendDocument( chat_id, document=open('{}-notimported.txt'.format(chat_id), 'rb'), caption=tl(update.effective_message, "*Data that can't be imported*"), timeout=360, parse_mode=ParseMode.MARKDOWN) os.remove("{}-notimported.txt".format(chat_id)) return except Exception as err: msg.reply_text(tl( update.effective_message, "An error has occurred getting Monica backup!\nGo, ping [my owner](https://t.me/kingofelephants) and ask if any solution of it!\n\nMaybe they can resolve your issue!" ), parse_mode="markdown") LOGGER.exception("An error when importing from Julie base!") return # only import one group if len(data) > 1 and str(chat.id) not in data: msg.reply_text( "There are more than one group in this file and the chat.id is not same! How am i supposed to import it?" ) return # Check if backup is this chat try: if data.get(str(chat.id)) == None: if conn: text = "Backup comes from another chat, I can't return another chat to chat *{}*".format( chat_name) else: text = "Backup comes from another chat, I can't return another chat to this chat" return msg.reply_text(text, parse_mode="markdown") except: return msg.reply_text( "There is problem while importing the data! Please ask in @HarukaAyaGroup about why this happened." ) # Check if backup is from self try: if str(bot.id) != str(data[str(chat.id)]['bot']): return msg.reply_text( "Backup from another bot that is not suggested might cause the problem, documents, photos, videos, audios, records might not work as it should be. However, You can still request a feature regarding this in @HarukaAyaGroup !" ) except: pass # Select data source if str(chat.id) in data: data = data[str(chat.id)]['hashes'] else: data = data[list(data.keys())[0]]['hashes'] try: for mod in DATA_IMPORT: mod.__import_data__(str(chat.id), data) except Exception: msg.reply_text( "An error occurred while recovering your data. The process failed. If you experience a problem with this, please ask in @HarukaAyaGroup . My owner and community will be happy to help. Also, bugs report makes me even better!\nThank you!" ) LOGGER.exception("Imprt for the chat %s with the name %s failed.", str(chat.id), str(chat.title)) return # TODO: some of that link logic # NOTE: consider default permissions stuff? if conn: text = "Backup fully restored on *{}*.".format(chat_name) else: text = "Backup fully restored" msg.reply_text(text, parse_mode="markdown")
async def analyze(request): img_data = await request.form() img_bytes = await (img_data['file'].read()) img = open_image(BytesIO(img_bytes)) prediction = learn.predict(img)[0] return JSONResponse({'result': str(prediction)})
class seek_wrapper: """Adds a seek method to a file object. This is only designed for seeking on readonly file-like objects. Wrapped file-like object must have a read method. The readline method is only supported if that method is present on the wrapped object. The readlines method is always supported. xreadlines and iteration are supported only for Python 2.2 and above. Public attributes: wrapped: the wrapped file object is_closed: true iff .close() has been called WARNING: All other attributes of the wrapped object (ie. those that are not one of wrapped, read, readline, readlines, xreadlines, __iter__ and next) are passed through unaltered, which may or may not make sense for your particular file object. """ # General strategy is to check that cache is full enough, then delegate to # the cache (self.__cache, which is a BytesIO instance). A seek # position (self.__pos) is maintained independently of the cache, in order # that a single cache may be shared between multiple seek_wrapper objects. # Copying using module copy shares the cache in this way. def __init__(self, wrapped): self.wrapped = wrapped self.__read_complete_state = [False] self.__is_closed_state = [False] self.__have_readline = hasattr(self.wrapped, "readline") self.__cache = BytesIO() self.__pos = 0 # seek position def invariant(self): # The end of the cache is always at the same place as the end of the # wrapped file (though the .tell() method is not required to be present # on wrapped file). return self.wrapped.tell() == len(self.__cache.getvalue()) def close(self): self.wrapped.close() self.is_closed = True def __getattr__(self, name): if name == "is_closed": return self.__is_closed_state[0] elif name == "read_complete": return self.__read_complete_state[0] wrapped = self.__dict__.get("wrapped") if wrapped: return getattr(wrapped, name) return getattr(self.__class__, name) def __setattr__(self, name, value): if name == "is_closed": self.__is_closed_state[0] = bool(value) elif name == "read_complete": if not self.is_closed: self.__read_complete_state[0] = bool(value) else: self.__dict__[name] = value def seek(self, offset, whence=0): assert whence in [0, 1, 2] # how much data, if any, do we need to read? if whence == 2: # 2: relative to end of *wrapped* file if offset < 0: raise ValueError("negative seek offset") # since we don't know yet where the end of that file is, we must # read everything to_read = None else: if whence == 0: # 0: absolute if offset < 0: raise ValueError("negative seek offset") dest = offset else: # 1: relative to current position pos = self.__pos if pos < offset: raise ValueError("seek to before start of file") dest = pos + offset end = len_of_seekable(self.__cache) to_read = dest - end if to_read < 0: to_read = 0 if to_read != 0: self.__cache.seek(0, 2) if to_read is None: assert whence == 2 self.__cache.write(self.wrapped.read()) self.read_complete = True self.__pos = self.__cache.tell() - offset else: data = self.wrapped.read(to_read) if not data: self.read_complete = True else: self.__cache.write(data) # Don't raise an exception even if we've seek()ed past the end # of .wrapped, since fseek() doesn't complain in that case. # Also like fseek(), pretend we have seek()ed past the end, # i.e. not: # self.__pos = self.__cache.tell() # but rather: self.__pos = dest else: self.__pos = dest def tell(self): return self.__pos def __copy__(self): cpy = self.__class__(self.wrapped) cpy.__cache = self.__cache cpy.__read_complete_state = self.__read_complete_state cpy.__is_closed_state = self.__is_closed_state return cpy def get_data(self): pos = self.__pos try: self.seek(0) return self.read(-1) finally: self.__pos = pos def read(self, size=-1): pos = self.__pos end = len_of_seekable(self.__cache) available = end - pos # enough data already cached? if size <= available and size != -1: self.__cache.seek(pos) self.__pos = pos + size return self.__cache.read(size) # no, so read sufficient data from wrapped file and cache it self.__cache.seek(0, 2) if size == -1: self.__cache.write(self.wrapped.read()) self.read_complete = True else: to_read = size - available assert to_read > 0 data = self.wrapped.read(to_read) if not data: self.read_complete = True else: self.__cache.write(data) self.__cache.seek(pos) data = self.__cache.read(size) self.__pos = self.__cache.tell() assert self.__pos == pos + len(data) return data def readline(self, size=-1): if not self.__have_readline: raise NotImplementedError("no readline method on wrapped object") # line we're about to read might not be complete in the cache, so # read another line first pos = self.__pos self.__cache.seek(0, 2) data = self.wrapped.readline() if not data: self.read_complete = True else: self.__cache.write(data) self.__cache.seek(pos) data = self.__cache.readline() if size != -1: r = data[:size] self.__pos = pos + size else: r = data self.__pos = pos + len(data) return r def readlines(self, sizehint=-1): pos = self.__pos self.__cache.seek(0, 2) self.__cache.write(self.wrapped.read()) self.read_complete = True self.__cache.seek(pos) data = self.__cache.readlines(sizehint) self.__pos = self.__cache.tell() return data def __iter__(self): return self def __next__(self): line = self.readline() if line == "": raise StopIteration return line next = __next__ xreadlines = __iter__ def __repr__(self): return ("<%s at %s whose wrapped object = %r>" % (self.__class__.__name__, hex(abs(id(self))), self.wrapped))
def send_prediction_on_photo(update, context): global storage global storage1 # Нам нужно получить две картинки, чтобы произвести перенос стиля, но каждая картинка приходит в # отдельном апдейте, поэтому в простейшем случае мы будем сохранять id первой картинки в память, # чтобы, когда уже придет вторая, мы могли загрузить в память уже сами картинки и обработать их. # Точно место для улучшения, я бы bot = context.bot if update.message.text == '/style': storage.append('1') update.message.reply_text( "Пришли 2 фотографии, первая фотография - то, что хочешь изменить. Вторая - стиль, который хочешь получить (любая картина художника)." ) elif len(storage) != 0: chat_id = update.message.chat_id print("Got image from {}".format(chat_id)) # получаем информацию о картинке image_info = update.message.photo[-1] image_file = bot.get_file(image_info) if chat_id in first_image_file: # первая картинка, которая к нам пришла станет content image, а вторая style image print("работает style_transfer") content_image_stream = BytesIO() first_image_file[chat_id].download(out=content_image_stream) del first_image_file[chat_id] style_image_stream = BytesIO() image_file.download(out=style_image_stream) output = model.transfer_style(content_image_stream, style_image_stream, num_steps=300) # теперь отправим назад фото output_stream = BytesIO() # output.save(output_stream, format='PNG') output_stream.seek(0) bot.send_photo(chat_id, photo=output_stream) print("Sent Photo to user") storage = [] else: first_image_file[chat_id] = image_file # if update.message.text == '/photo_real': storage1.append('1') update.message.reply_text( "Пришли 2 фотографии, первая фотография - пейзаж. Вторая - стиль, который хочешь перенести на свою фотографию." ) elif len(storage1) != 0: chat_id = update.message.chat_id print("Got image from {}".format(chat_id)) # получаем информацию о картинке image_info = update.message.photo[-1] image_file = bot.get_file(image_info) if chat_id in first_image_file: # первая картинка, которая к нам пришла станет content image, а вторая style image print("работает photo_real") content_image_stream = BytesIO() first_image_file[chat_id].download(out=content_image_stream) del first_image_file[chat_id] style_image_stream = BytesIO() image_file.download(out=style_image_stream) process_stylization.stylization( stylization_module=p_wct, smoothing_module=p_pro, content_image_path=content_image_stream, style_image_path=style_image_stream, content_seg_path=[], style_seg_path=[], output_image_path='img.jpg', cuda=False, save_intermediate=False, no_post=False) output = Image.open('img.jpg') # теперь отправим назад фото output_stream = BytesIO() # output.save(output_stream, format='PNG') output_stream.seek(0) bot.send_photo(chat_id, photo=output_stream) storage1 = [] print("Sent Photo to user") else: first_image_file[chat_id] = image_file #
def _load(self): self.document = WordprocessingDocument(path=None) package = self.document.package document_part = package.create_part(uri='/word/document.xml', ) if self.styles_xml: self.relationships.append({ 'external': False, 'target_path': 'styles.xml', 'data': self.styles_xml, 'relationship_id': 'styles', 'relationship_type': StyleDefinitionsPart.relationship_type, }) for relationship in self.relationships: target_mode = 'Internal' if relationship['external']: target_mode = 'External' target_uri = relationship['target_path'] if 'data' in relationship: full_target_uri = posixpath.join( package.uri, 'word', target_uri, ) package.streams[full_target_uri] = BytesIO( relationship['data'], ) package.create_part(uri=full_target_uri) document_part.create_relationship( target_uri=target_uri, target_mode=target_mode, relationship_type=relationship['relationship_type'], relationship_id=relationship['relationship_id'], ) package.streams[document_part.uri] = BytesIO(self.document_xml) package.create_relationship( target_uri=document_part.uri, target_mode='Internal', relationship_type=MainDocumentPart.relationship_type, ) self.numbering_root = None if self.numbering_dict is not None: self.numbering_root = parse_xml_from_string( DXB.numbering(self.numbering_dict), ) # This is the standard page width for a word document (in points), Also # the page width that we are looking for in the test. self.page_width = 612 self.styles_manager = StylesManager( self.document.main_document_part.style_definitions_part, ) self.styles = self.styles_manager.styles self.parse_begin(self.document.main_document_part.root_element)
async def cameraLoop(): global continueLoop global freshParams await asyncio.sleep(1) while True: try: log = logging.getLogger("cameraLoop") params = cleanParams(freshParams) shootresol = params["shootresol"] strtResolution = (shootresol["width"], shootresol["height"]) log.info("OpeningCamera at resol %s", (strtResolution, )) with PiCamera(resolution=strtResolution, framerate_range=(0.1, 30)) as camera: log.info("got camera") continueLoop = True while continueLoop: params = cleanParams(freshParams) shootresol = params["shootresol"] dispresol = (params["dispresol"]["width"], params["dispresol"]["height"]) if (shootresol["width"] != strtResolution[0]): log.info("resolution changed!") break log.info("looping %s", params) capture_format = params["capture_format"] # setting camera config camsetting_start = time.time() camera.iso = params["isovalue"] camera.brightness = params["brightness"] camera.contrast = params["contrast"] # camera.analog_gain=1.0 camera.exposure_mode = params["expomode"] camera.exposure_compensation = params[ "exposure_compensation"] camera.saturation = params["saturation"] camera.shutter_speed = params["shutterSpeed"] log.info("setting awb_mode off") camera.awb_mode = 'off' g = (params["redgain"], params["bluegain"]) camera.awb_gains = g camsetting_dur = time.time() - camsetting_start log.info("create databuff") if (capture_format in ["yuv"]): stream = open("/dev/shm/lol.data", "w+b") else: stream = BytesIO() log.info("capture start capture_format:%s", capture_format) triggerDate = time.time() camera.capture(stream, format=capture_format, use_video_port=False) capture_dur = time.time() - triggerDate # into pill log.info("reading image") strtTime = time.time() if (capture_format == "jpeg"): stream.seek(0) image = Image.open(stream) elif (capture_format == "yuv"): stream.seek(0) rgb = imgutils.yuvbytesToRgb(stream, *strtResolution) image = Image.fromarray(rgb) elif (capture_format == "rgb"): stream.seek(0) image = Image.frombytes("RGB", strtResolution, stream.getvalue(), "raw", "RGB", 0, 1) #image = Image.frombuffer("RGB",strtResolution,stream,"raw","RGB",0,1) pil_dur = time.time() - strtTime #log.info(image.size) strtTime = time.time() #histData = imgutils.colorHist(image) hist_dur = time.time() - strtTime log.info("resizing") strtTime = time.time() imageDisplay = imgutils.resizeImage(image, dispresol) resize_dur = time.time() - strtTime # save original image strtTime = time.time() save_format = params["save_format"] save_section = params["save_section"] save_subsection = params["save_subsection"] if save_format in [ "none", ]: pass else: IMGBUFF.stack((image, save_format, save_section, save_subsection, triggerDate)) save_dur = time.time() - strtTime # publish image to server usedParams = { "triggerDate": triggerDate, "gains": [float(_) for _ in camera.awb_gains], "analog_gain": float(camera.analog_gain), "iso": camera.iso, "brightness": camera.brightness, "saturation": camera.saturation, "contrast": camera.contrast, "exposure_compensation": camera.exposure_compensation, "resolution": list(strtResolution), "imageSize": image.size, "shutterSpeed": camera.shutter_speed, "exposure_speed": camera.exposure_speed, "exposure_mode": camera.exposure_mode, "awb_mode": camera.awb_mode, "capture_format": capture_format, "save_format": save_format, "save_section": save_section, "camsetting_dur": camsetting_dur, "capture_dur": capture_dur, "pil_dur": pil_dur, "hist_dur": hist_dur, "resize_dur": resize_dur, "save_dur": save_dur, } strtTime = time.time() if serverConnection: try: data = imgutils.pilimTobase64Jpg(imageDisplay) msg = json.dumps({ "usedParams": usedParams, "msgtype": "srcimage", "imageData": data }) await serverConnection.send(msg) except Exception as e: log.exception("err %s", e) else: log.info("no server") # end sending send_dur = time.time() - strtTime # now send timing data timingData = { "triggerDate": triggerDate, "imgbuffcount": len(IMGBUFF.content), "send_dur": send_dur, "usedParams": usedParams } msg = ",".join([ "%s: %.2f" % (k, timingData[k]) for k in sorted(timingData.keys()) if "dur" in k ]) log.info("timing info %s , %s in buff", msg, len(IMGBUFF.content)) if serverConnection: msg = makeMessage("camTiming", timingData, jdump=True) await serverConnection.send(msg) await asyncio.sleep(.0001) # end while log.info("ending capture loop, camera will be reopened") #end with camera #no camera here log.info("camera closed") except Exception as e: log.exception("whooops") await asyncio.sleep(1)
def test_stdin_binary_mode(self): with mock.patch('sys.stdin') as m_stdin: m_stdin.buffer = BytesIO(b'spam, bacon, sausage, and spam') fi = FileInput(files=['-'], mode='rb') lines = list(fi) self.assertEqual(lines, [b'spam, bacon, sausage, and spam'])
def FromHex(obj, hex_string): obj.deserialize(BytesIO(hex_str_to_bytes(hex_string))) return obj
def download_file(url: str) -> str: resp = requests.get(url) with zipfile.ZipFile(BytesIO(resp.content)) as zip_ref: zip_ref.extractall(FILES_DIR) return url
import subprocess import sys dataSplit = recvData.split() raspberryPiID = dataSplit[0] cameraID = dataSplit[1] photoID = dataSplit[2] num_df = pd.read_sql(sql="select max(num) from images", con=engine) num = num_df.values[0][0] img_df = pd.read_sql(sql="select * from images where num = " + str(num), con=engine) img_str = img_df['data'].values[0] img = base64.decodebytes(img_str) im = Image.open(BytesIO(img)) tempname = 'temp.jpg' im.save('imgs/' + tempname) filename = glob('detectplate.py') subprocess.call([ 'python', filename, '--images', 'imgs/' + tempname, '--det', 'det', '--photoID', photoID, '--cameraID', cameraID ]) buffer = BytesIO() im = Image.open('det/det_' + tempname) im.save(buffer, format='jpeg') img_str2 = base64.b64encode(buffer.getvalue())
def assertPixelColor(self, response, x, y, color): img = Image.open(BytesIO(response.content)) self.assertEqual(color, img.getpixel((x, y)))
def log(s): # print(s) f = open('logs/events.log', 'a') date = datetime.now().strftime("%Y-%m-%d\t%H:%M:%S") f.writelines(f"[{date}]\t{s}\n") f.close() log("[IMPORTING]\tImports Done!") lemmatizer = WordNetLemmatizer() intents = json.loads(open('src/intents.json').read()) words = pickle.load(open('Data/words.pkl', 'rb')) classes = pickle.load(open('Data/classes.pkl', 'rb')) mp3_fp = BytesIO() try: model = load_model('Data/model.h5') log("[SUCCESS]\tModel loaded.") except: log("[FAILED]\tModel not found.") log("[PROCESSING]\tTraining model.") def speak(audio): tts = gTTS(audio, lang='en', tld='co.uk') tts.save("Data/temp.mp3") os.system("mpg123 -q Data/temp.mp3")
def test_formdata_file_upload_missing_param(simple_app): app_client = simple_app.app.test_client() resp = app_client.post( '/v1.0/test-formData-file-upload-missing-param', data={'missing_formData': (BytesIO(b'file contents'), 'example.txt')}) assert resp.status_code == 200
def createPDF(datasetID): """ Get the ID of the dataset protocol and creates a PDF of this protocol using the reportlab tools :param datasetID: :return: a HTTPResponse PDF object """ # Get all the information of this protocol basicInfo = BasicDataset.objects.get(id=datasetID) partnerInfo = Partner.objects.filter(dataset_id=datasetID) reqInfo = DataReq.objects.filter(dataset_id=datasetID).order_by('taskNr') stepInfo = ExpStep.objects.filter(dataset_id=datasetID).order_by('taskNr') reportingInfo = Reporting.objects.filter(dataset_id=datasetID).order_by('taskNr') # Create the HttpResponse object with the appropriate PDF headers. response = HttpResponse(content_type='application/pdf') response['Content-Disposition'] = 'attachment; filename=' + basicInfo.shortname + '.pdf' styles = { 'default': ParagraphStyle( 'default', fontName='Times-Roman', fontSize=10, leading=11, leftIndent=0, rightIndent=0, firstLineIndent=0, spaceBefore=0, spaceAfter=0, textColor= black, backColor=None, wordWrap=None, borderWidth= 0, borderPadding= 0, borderColor= None, borderRadius= None, allowWidows= 1, allowOrphans= 0, endDots=None, splitLongWords=1, ), } styles['title1'] = ParagraphStyle( 'title1', parent=styles['default'], fontName='Helvetica-Bold', fontSize=24, leading=30, alignment=TA_CENTER, textColor=black, ) styles['title2'] = ParagraphStyle( 'title2', parent=styles['default'], fontName='Helvetica', fontSize=14, leading=18, leftIndent=5, alignment=TA_LEFT, textColor=black, spaceBefore = 0, borderRadius = None, firstLineIndent = 0, underlineProportion = 0.0, rightIndent = 0, wordWrap = None, allowWidows = 1, backColor = Color(.9,.9,.9), justifyLastLine = 0, textTransform = None, justifyBreaks = 0, spaceShrinkage = 0.05, splitLongWords = 1, bulletFontSize = 10, borderWidth = 1, borderPadding = 2, endDots = None, spaceAfter = 6, ) styles['label'] = ParagraphStyle( 'label', parent=styles['default'], fontName='Times-Bold', ) buffer = BytesIO() doc = BaseDocTemplate(buffer) doc.addPageTemplates( [ PageTemplate( frames=[ Frame( doc.leftMargin, doc.bottomMargin, doc.width, doc.height, id=None ), ] ), ] ) # container for the 'Flowable' objects story = [] scriptDir = os.path.dirname(__file__) im = Image(os.path.join(scriptDir, "static/img/sologo_new_cropped.png"), width=2.6*cm, height=2*cm) im.hAlign = 'RIGHT' data = [[im, Paragraph('{}'.format(basicInfo.shortname), styles['title1'])]] t=Table(data, hAlign='LEFT', colWidths=[2 * cm, 14 * cm]) t.setStyle(TableStyle([('VALIGN',(0,0),(-1,-1),'TOP')])) story.append(t) story.append(Spacer(1, 16)) story.append(Paragraph('Experiment Information', styles['title2'])) data= [[Paragraph('Full experiment name:', styles['label']), Paragraph(basicInfo.title, styles['default'])], [Paragraph('Experiment Idea:', styles['label']), Paragraph(basicInfo.experimentIdea.replace('\n','<br />\n'), styles['default'])], [Paragraph('Hypothesis:', styles['label']), Paragraph(basicInfo.hypothesis.replace('\n','<br />\n'), styles['default'])], [Paragraph('Research objective:', styles['label']), Paragraph(basicInfo.researchObjective.replace('\n','<br />\n'), styles['default'])]] t=Table(data, hAlign='LEFT', colWidths=[4 * cm, 12 * cm]) t.setStyle(TableStyle([('VALIGN',(0,0),(-1,-1),'TOP')])) story.append(t) story.append(Spacer(1, 24)) # Partners story.append(Paragraph('Partners', styles['title2'])) for partner in partnerInfo: partnerName = Paragraph('{}'.format(partner.name), styles['default']) if partner.lead == True: partnerName = Paragraph('{} (lead)'.format(partner.name), styles['default']) data = [[Paragraph('Name:', styles['label']), partnerName], [Paragraph('E-mail:', styles['label']), Paragraph(partner.email, styles['default'])], [Paragraph('Organisation:', styles['label']), Paragraph(partner.organisation, styles['default'])]] t=Table(data, hAlign='LEFT', colWidths=[4 * cm, 12 * cm]) t.setStyle(TableStyle([('VALIGN',(0,0),(-1,-1),'TOP')])) story.append(t) story.append(Spacer(1, 16)) story.append(Spacer(1, 24)) story.append(Paragraph('A) Data & Method Preparation', styles['title2'])) writeTasks(story, styles, reqInfo) story.append(Spacer(1, 24)) story.append(Paragraph('B) Experiment Analysis Steps', styles['title2'])) writeTasks(story, styles, stepInfo) story.append(Spacer(1, 24)) story.append(Paragraph('C) Result Reporting', styles['title2'])) writeTasks(story, styles, reportingInfo) # write the document to disk doc.build(story) pdf = buffer.getvalue() buffer.close() response.write(pdf) return response
def main (): print("Confluent Kafka Version: %s - Libversion: %s" % (version(), libversion())) print("") print("Using weights file: %s" % weights_file) print("") print("Consuming raw video frames from %s" % topic_frames) print("") print("Producing classified video frames to: %s" % topic_class_frames) print("") print("Consumer group name: %s" % consumer_group) print("") print("Consumer group start: %s" % consumer_group_start) print("") print("Debug is set to %s" % debug) print("") print("nomsgcnt is set to %s - if this is greater than 0, then when we have that many attempts to read a msg from MapR Streams, we will exit" % nomsgcnt) con_conf = {'bootstrap.servers': '', 'group.id': consumer_group, 'default.topic.config': {'auto.offset.reset': consumer_group_start}} pro_conf = {'bootstrap.servers': '', 'message.max.bytes':'2978246'} c = Consumer(con_conf) p = Producer(pro_conf) c.subscribe([topic_frames]) lastmsgtime = time.time() nomsg = 0 running = True while running: msg = c.poll(timeout=1.0) if msg is None: nomsg += 1 if debug: print("No Message - Continuing") if nomsgcnt > 0 and nomsg >= nomsgcnt: print("%s itterations with no messages reached - Exiting Gracefully") sys.exit(0) continue if not msg.error(): mymsg = json.loads(msg.value().decode('utf-8'), object_pairs_hook=OrderedDict) mypart = msg.partition() myoffset = msg.offset() outmsg = OrderedDict() outmsg['ts'] = mymsg['ts'] outmsg['epoch_ts'] = mymsg['epoch_ts'] outmsg['cam_name'] = mymsg['cam_name'] outmsg['src_partition'] = mypart outmsg['src_offset'] = myoffset mybytes = base64.b64decode(mymsg['img']) o = open("/dev/shm/tmp.jpg", "wb") o.write(mybytes) o.close # myimage = np.array(Image.open(BytesIO(mybytes))) curmsgtime = time.time() msgdelta = curmsgtime - lastmsgtime if debug: print("Time between last processed messages: %s" % msgdelta) lastmsgtime = curmsgtime r = python.darknet.detect(net, meta, b'/dev/shm/tmp.jpg') if r != []: if debug: print("Got classification!") curtime = datetime.datetime.now() mystrtime = curtime.strftime("%Y-%m-%d %H:%M:%S") epochtime = int(time.time()) arclass = [] if save_images == 1: try: image = Image.open(BytesIO(mybytes)).convert("RGBA") except: continue draw = ImageDraw.Draw(image) for q in r: j = OrderedDict() name = q[0] j['name'] = name.decode() predict = q[1] j['predict'] = predict x = q[2][0] y = q[2][1] w = q[2][2] z = q[2][3] x_max = (2*x+w)/2 x_min = (2*x-w)/2 y_min = (2*y-z)/2 y_max = (2*y+z)/2 j['x_min'] = x_min j['x_max'] = x_max j['y_min'] = y_min j['y_max'] = y_max for x in range(border): draw.rectangle(((x_min - x, y_min - x), (x_max + x, y_max + x)), fill=None, outline="black") draw.text((x_min + border + 2, y_max - border - 5), name) arclass.append(j) imgSave = BytesIO() image.save(imgSave, format='JPEG') imgSave = imgSave.getvalue() encdata = base64.b64encode(imgSave) encdatastr = encdata.decode('utf-8') else: encdatastr = "" outmsg['class_json'] = arclass outmsg['class_ts'] = mystrtime outmsg['class_epoch_ts'] = epochtime outmsg['class_img'] = encdatastr produceMessage(p, topic_class_frames, json.dumps(outmsg)) else: pass
def _find_module_utils(module_name, b_module_data, module_path, module_args, task_vars, module_compression, async_timeout, become, become_method, become_user, become_password, environment): """ Given the source of the module, convert it to a Jinja2 template to insert module code and return whether it's a new or old style module. """ module_substyle = module_style = 'old' # module_style is something important to calling code (ActionBase). It # determines how arguments are formatted (json vs k=v) and whether # a separate arguments file needs to be sent over the wire. # module_substyle is extra information that's useful internally. It tells # us what we have to look to substitute in the module files and whether # we're using module replacer or ansiballz to format the module itself. if _is_binary(b_module_data): module_substyle = module_style = 'binary' elif REPLACER in b_module_data: # Do REPLACER before from ansible.module_utils because we need make sure # we substitute "from ansible.module_utils basic" for REPLACER module_style = 'new' module_substyle = 'python' b_module_data = b_module_data.replace(REPLACER, b'from ansible.module_utils.basic import *') elif b'from ansible.module_utils.' in b_module_data: module_style = 'new' module_substyle = 'python' elif REPLACER_WINDOWS in b_module_data or re.search(b'#Requires \-Module', b_module_data, re.IGNORECASE): module_style = 'new' module_substyle = 'powershell' elif REPLACER_JSONARGS in b_module_data: module_style = 'new' module_substyle = 'jsonargs' elif b'WANT_JSON' in b_module_data: module_substyle = module_style = 'non_native_want_json' shebang = None # Neither old-style, non_native_want_json nor binary modules should be modified # except for the shebang line (Done by modify_module) if module_style in ('old', 'non_native_want_json', 'binary'): return b_module_data, module_style, shebang output = BytesIO() py_module_names = set() if module_substyle == 'python': params = dict(ANSIBLE_MODULE_ARGS=module_args,) python_repred_params = repr(json.dumps(params)) try: compression_method = getattr(zipfile, module_compression) except AttributeError: display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression) compression_method = zipfile.ZIP_STORED lookup_path = os.path.join(C.DEFAULT_LOCAL_TMP, 'ansiballz_cache') cached_module_filename = os.path.join(lookup_path, "%s-%s" % (module_name, module_compression)) zipdata = None # Optimization -- don't lock if the module has already been cached if os.path.exists(cached_module_filename): display.debug('ANSIBALLZ: using cached module: %s' % cached_module_filename) zipdata = open(cached_module_filename, 'rb').read() else: if module_name in action_write_locks.action_write_locks: display.debug('ANSIBALLZ: Using lock for %s' % module_name) lock = action_write_locks.action_write_locks[module_name] else: # If the action plugin directly invokes the module (instead of # going through a strategy) then we don't have a cross-process # Lock specifically for this module. Use the "unexpected # module" lock instead display.debug('ANSIBALLZ: Using generic lock for %s' % module_name) lock = action_write_locks.action_write_locks[None] display.debug('ANSIBALLZ: Acquiring lock') with lock: display.debug('ANSIBALLZ: Lock acquired: %s' % id(lock)) # Check that no other process has created this while we were # waiting for the lock if not os.path.exists(cached_module_filename): display.debug('ANSIBALLZ: Creating module') # Create the module zip data zipoutput = BytesIO() zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method) # Note: If we need to import from release.py first, # remember to catch all exceptions: https://github.com/ansible/ansible/issues/16523 zf.writestr('ansible/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n__version__="' + to_bytes(__version__) + b'"\n__author__="' + to_bytes(__author__) + b'"\n') zf.writestr('ansible/module_utils/__init__.py', b'from pkgutil import extend_path\n__path__=extend_path(__path__,__name__)\n') zf.writestr('ansible_module_%s.py' % module_name, b_module_data) py_module_cache = {('__init__',): (b'', '[builtin]')} recursive_finder(module_name, b_module_data, py_module_names, py_module_cache, zf) zf.close() zipdata = base64.b64encode(zipoutput.getvalue()) # Write the assembled module to a temp file (write to temp # so that no one looking for the file reads a partially # written file) if not os.path.exists(lookup_path): # Note -- if we have a global function to setup, that would # be a better place to run this os.makedirs(lookup_path) display.debug('ANSIBALLZ: Writing module') with open(cached_module_filename + '-part', 'wb') as f: f.write(zipdata) # Rename the file into its final position in the cache so # future users of this module can read it off the # filesystem instead of constructing from scratch. display.debug('ANSIBALLZ: Renaming module') os.rename(cached_module_filename + '-part', cached_module_filename) display.debug('ANSIBALLZ: Done creating module') if zipdata is None: display.debug('ANSIBALLZ: Reading module after lock') # Another process wrote the file while we were waiting for # the write lock. Go ahead and read the data from disk # instead of re-creating it. try: zipdata = open(cached_module_filename, 'rb').read() except IOError: raise AnsibleError('A different worker process failed to create module file. ' 'Look at traceback for that process for debugging information.') zipdata = to_text(zipdata, errors='surrogate_or_strict') shebang, interpreter = _get_shebang(u'/usr/bin/python', task_vars) if shebang is None: shebang = u'#!/usr/bin/python' # Enclose the parts of the interpreter in quotes because we're # substituting it into the template as a Python string interpreter_parts = interpreter.split(u' ') interpreter = u"'{0}'".format(u"', '".join(interpreter_parts)) now = datetime.datetime.utcnow() output.write(to_bytes(ACTIVE_ANSIBALLZ_TEMPLATE % dict( zipdata=zipdata, ansible_module=module_name, params=python_repred_params, shebang=shebang, interpreter=interpreter, coding=ENCODING_STRING, year=now.year, month=now.month, day=now.day, hour=now.hour, minute=now.minute, second=now.second, ))) b_module_data = output.getvalue() elif module_substyle == 'powershell': # Powershell/winrm don't actually make use of shebang so we can # safely set this here. If we let the fallback code handle this # it can fail in the presence of the UTF8 BOM commonly added by # Windows text editors shebang = u'#!powershell' exec_manifest = dict( module_entry=to_text(base64.b64encode(b_module_data)), powershell_modules=dict(), module_args=module_args, actions=['exec'], environment=environment ) exec_manifest['exec'] = to_text(base64.b64encode(to_bytes(leaf_exec))) if async_timeout > 0: exec_manifest["actions"].insert(0, 'async_watchdog') exec_manifest["async_watchdog"] = to_text(base64.b64encode(to_bytes(async_watchdog))) exec_manifest["actions"].insert(0, 'async_wrapper') exec_manifest["async_wrapper"] = to_text(base64.b64encode(to_bytes(async_wrapper))) exec_manifest["async_jid"] = str(random.randint(0, 999999999999)) exec_manifest["async_timeout_sec"] = async_timeout if become and become_method == 'runas': exec_manifest["actions"].insert(0, 'become') exec_manifest["become_user"] = become_user exec_manifest["become_password"] = become_password exec_manifest["become"] = to_text(base64.b64encode(to_bytes(become_wrapper))) lines = b_module_data.split(b'\n') module_names = set() requires_module_list = re.compile(to_bytes(r'(?i)^#\s*requires\s+\-module(?:s?)\s*(Ansible\.ModuleUtils\..+)')) for line in lines: # legacy, equivalent to #Requires -Modules powershell if REPLACER_WINDOWS in line: module_names.add(b'Ansible.ModuleUtils.Legacy') line_match = requires_module_list.match(line) if line_match: module_names.add(line_match.group(1)) for m in set(module_names): m = to_text(m) mu_path = ps_module_utils_loader.find_plugin(m, ".psm1") if not mu_path: raise AnsibleError('Could not find imported module support code for \'%s\'.' % m) exec_manifest["powershell_modules"][m] = to_text( base64.b64encode( to_bytes( _slurp(mu_path) ) ) ) # FUTURE: smuggle this back as a dict instead of serializing here; the connection plugin may need to modify it module_json = json.dumps(exec_manifest) b_module_data = exec_wrapper.replace(b"$json_raw = ''", b"$json_raw = @'\r\n%s\r\n'@" % to_bytes(module_json)) elif module_substyle == 'jsonargs': module_args_json = to_bytes(json.dumps(module_args)) # these strings could be included in a third-party module but # officially they were included in the 'basic' snippet for new-style # python modules (which has been replaced with something else in # ansiballz) If we remove them from jsonargs-style module replacer # then we can remove them everywhere. python_repred_args = to_bytes(repr(module_args_json)) b_module_data = b_module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) b_module_data = b_module_data.replace(REPLACER_COMPLEX, python_repred_args) b_module_data = b_module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) # The main event -- substitute the JSON args string into the module b_module_data = b_module_data.replace(REPLACER_JSONARGS, module_args_json) facility = b'syslog.' + to_bytes(task_vars.get('ansible_syslog_facility', C.DEFAULT_SYSLOG_FACILITY), errors='surrogate_or_strict') b_module_data = b_module_data.replace(b'syslog.LOG_USER', facility) return (b_module_data, module_style, shebang)
def write(self) -> bytes: b = BytesIO() b.write(Int(self.ID, False)) # No flags b.write(String(self.phone_number)) b.write(String(self.phone_code_hash)) b.write(String(self.phone_code)) return b.getvalue()
def write(self) -> bytes: b = BytesIO() b.write(Int(self.ID, False)) # No flags b.write(Int(self.id)) b.write(Long(self.access_hash)) b.write(Int(self.date)) b.write(Int(self.admin_id)) b.write(Int(self.participant_id)) b.write(Bytes(self.g_a_or_b)) b.write(Long(self.key_fingerprint)) return b.getvalue()
def doit(directory, force, force_resize, dry_run): """Do the thing. """ orig_dir = directory / "originals" if not orig_dir.exists(): print("Expected to find a sub-directory named 'originals' " "containing image files.", file=sys.stderr) sys.exit(1) content_file = directory / CONTENT_FILE if not content_file.exists(): content = [] else: with content_file.open() as cf: content = json.load(cf) old = {c['filename']: c for c in content} new = [] done = [] for path in sorted(orig_dir.glob('*.jpg')): if path.is_file(): # A candidate mtime = isoformat(path.stat().st_mtime) oe = old.get(path.name) if (force or oe is None or oe.get('mtime') != mtime): # File is new or changed new.append(dict( filename=path.name, mtime=mtime, path=path, ID=re.sub(r'[^\w-]', '-', path.stem), downloadURL=f"/images/{orig_dir.name}/{path.name}")) else: # We have up-to-date info for this file done.append(oe) if len(new) == 0: logging.info("No changes, exiting.") return resized_dir = directory / "resized" if not resized_dir.exists(): resized_dir.mkdir() # Process new files for data in new: path = data.pop("path") logging.info("Processing %s", path.name) img = Image.open(path) # Save the EXIF data so we can write it back out exif_bytes = img.info.get('exif', b'') if img.width > MAX_WIDTH or img.height > MAX_HEIGHT: # Image too large, need maxpect image for web display logging.info("Image too large (%d x %d)", img.width, img.height) resized_name = f"web-{path.name}" resized_path = resized_dir / resized_name if resized_path.exists() and not force_resize: logging.info("Reading size of existing maxpect") maxpect = Image.open(resized_path) else: logging.info("Making maxpect") maxpect = img.copy() # thumbnail() method modifies image, preserves aspect ratio. # Image.LANCZOS is the best quality and seems plenty fast # Image.BICUBIC is faster but lower quality. maxpect.thumbnail( (MAX_WIDTH, MAX_HEIGHT), resample=Image.LANCZOS) logging.debug('Saving maxpect as "%s"', resized_path) if not dry_run: maxpect.save(resized_path, quality=90, progressive=True, optimized=True, exif=exif_bytes, icc_profile=img.info.get('icc_profile')) data["imgWidth"] = maxpect.width data["imgHeight"] = maxpect.height data["src"] = f'/images/{resized_dir.name}/{resized_name}' else: data["src"] = data["downloadURL"] data["imgWidth"] = img.width data["imgHeight"] = img.height read_exif_metadata(img, data) if "title" not in data: # Nothing in EXIF, use the filename if not re.search(r'\d{5}', path.name): # Doesn't look like a serial number, assume it's text and try # to make it pretty. data['title'] = titlecase(re.sub(r'[_-]', ' ', path.stem)) else: data['title'] = path.name # make thumbnail (cropping to 90%) thumb_path = resized_dir / f"thumb-{path.name}" logging.info("Making thumbnail %s", thumb_path) crop_coords = ( img.width / 20, img.height / 20, img.width - img.width / 20, img.height - img.height / 20 ) thumb = img.crop(crop_coords) hratio = thumb.height / THUMBNAIL_HEIGHT thumb.thumbnail((thumb.width * hratio, THUMBNAIL_HEIGHT)) if not dry_run: thumb.save(thumb_path) data["srct"] = f"/images/{resized_dir.name}/{thumb_path.name}" data["imgtWidth"] = thumb.width data["imgtHeight"] = thumb.height # Get dominant colors # Resize to ~20x20, blur, create gif, base64 encode # (Fancier method: https://github.com/fengsp/color-thief-py) logging.info("Creating 'dominant colors' gif") thumb.thumbnail((15, 15)) blurred = thumb.filter(filter=ImageFilter.BLUR) bio = BytesIO() blurred.save(bio, format="GIF") gif_encoded = binascii.b2a_base64(bio.getvalue()).decode('utf8') # Add to new dict data['imageDominantColors'] = f"data:image/gif;base64,{gif_encoded}" done.append(data) # FIXME Remove orphaned thumbs and originals # Write new CONTENT_FILE done.sort(key=lambda x: x.get('exifTime', x['mtime']), reverse=True) if dry_run: print(json.dumps(done, indent=1), file=sys.stderr) else: # Make symlink to latest thumbnail image latest = Path(done[0]['srct']).name symlink_path = resized_dir / 'latest.jpg' try: if symlink_path.exists() or symlink_path.is_symlink(): logging.debug("unlinking old symlink %s", symlink_path) symlink_path.unlink() logging.info("Creating 'latest.jpg' symlink %s -> %s", symlink_path, latest) (symlink_path).symlink_to(latest) except OSError as e: logging.error("Failed to create 'latest.jpg' symlink: " + str(e)) # Write JSON logging.info("Writing %s", directory / CONTENT_FILE) with (directory / CONTENT_FILE).open(mode='w') as fp: json.dump(done, fp, indent=1)
def decoderawtransaction_asm_sighashtype(self): """Test decoding scripts via RPC command "decoderawtransaction". This test is in with the "decodescript" tests because they are testing the same "asm" script decodes. """ # this test case uses a random plain vanilla mainnet transaction with a single P2PKH input and output tx = '0100000001696a20784a2c70143f634e95227dbdfdf0ecd51647052e70854512235f5986ca010000008a47304402207174775824bec6c2700023309a168231ec80b82c6069282f5133e6f11cbb04460220570edc55c7c5da2ca687ebd0372d3546ebc3f810516a002350cac72dfe192dfb014104d3f898e6487787910a690410b7a917ef198905c27fb9d3b0a42da12aceae0544fc7088d239d9a48f2828a15a09e84043001f27cc80d162cb95404e1210161536ffffffff0100e1f505000000001976a914eb6c6e0cdb2d256a32d97b8df1fc75d1920d9bca88ac00000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('304402207174775824bec6c2700023309a168231ec80b82c6069282f5133e6f11cbb04460220570edc55c7c5da2ca687ebd0372d3546ebc3f810516a002350cac72dfe192dfb[ALL] 04d3f898e6487787910a690410b7a917ef198905c27fb9d3b0a42da12aceae0544fc7088d239d9a48f2828a15a09e84043001f27cc80d162cb95404e1210161536', rpc_result['vin'][0]['scriptSig']['asm']) # this test case uses a mainnet transaction that has a P2SH input and both P2PKH and P2SH outputs. # it's from James D'Angelo's awesome introductory videos about multisig: https://www.youtube.com/watch?v=zIbUSaZBJgU and https://www.youtube.com/watch?v=OSA1pwlaypc # verify that we have not altered scriptPubKey decoding. tx = '01000000018d1f5635abd06e2c7e2ddf58dc85b3de111e4ad6e0ab51bb0dcf5e84126d927300000000fdfe0000483045022100ae3b4e589dfc9d48cb82d41008dc5fa6a86f94d5c54f9935531924602730ab8002202f88cf464414c4ed9fa11b773c5ee944f66e9b05cc1e51d97abc22ce098937ea01483045022100b44883be035600e9328a01b66c7d8439b74db64187e76b99a68f7893b701d5380220225bf286493e4c4adcf928c40f785422572eb232f84a0b83b0dea823c3a19c75014c695221020743d44be989540d27b1b4bbbcfd17721c337cb6bc9af20eb8a32520b393532f2102c0120a1dda9e51a938d39ddd9fe0ebc45ea97e1d27a7cbd671d5431416d3dd87210213820eb3d5f509d7438c9eeecb4157b2f595105e7cd564b3cdbb9ead3da41eed53aeffffffff02611e0000000000001976a914dc863734a218bfe83ef770ee9d41a27f824a6e5688acee2a02000000000017a9142a5edea39971049a540474c6a99edf0aa4074c588700000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('8e3730608c3b0bb5df54f09076e196bc292a8e39a78e73b44b6ba08c78f5cbb0', rpc_result['txid']) assert_equal('0 3045022100ae3b4e589dfc9d48cb82d41008dc5fa6a86f94d5c54f9935531924602730ab8002202f88cf464414c4ed9fa11b773c5ee944f66e9b05cc1e51d97abc22ce098937ea[ALL] 3045022100b44883be035600e9328a01b66c7d8439b74db64187e76b99a68f7893b701d5380220225bf286493e4c4adcf928c40f785422572eb232f84a0b83b0dea823c3a19c75[ALL] 5221020743d44be989540d27b1b4bbbcfd17721c337cb6bc9af20eb8a32520b393532f2102c0120a1dda9e51a938d39ddd9fe0ebc45ea97e1d27a7cbd671d5431416d3dd87210213820eb3d5f509d7438c9eeecb4157b2f595105e7cd564b3cdbb9ead3da41eed53ae', rpc_result['vin'][0]['scriptSig']['asm']) assert_equal('OP_DUP OP_HASH160 dc863734a218bfe83ef770ee9d41a27f824a6e56 OP_EQUALVERIFY OP_CHECKSIG', rpc_result['vout'][0]['scriptPubKey']['asm']) assert_equal('OP_HASH160 2a5edea39971049a540474c6a99edf0aa4074c58 OP_EQUAL', rpc_result['vout'][1]['scriptPubKey']['asm']) txSave = CTransaction() txSave.deserialize(BytesIO(hex_str_to_bytes(tx))) # make sure that a specifically crafted op_return value will not pass all the IsDERSignature checks and then get decoded as a sighash type tx = '01000000015ded05872fdbda629c7d3d02b194763ce3b9b1535ea884e3c8e765d42e316724020000006b48304502204c10d4064885c42638cbff3585915b322de33762598321145ba033fc796971e2022100bb153ad3baa8b757e30a2175bd32852d2e1cb9080f84d7e32fcdfd667934ef1b012103163c0ff73511ea1743fb5b98384a2ff09dd06949488028fd819f4d83f56264efffffffff0200000000000000000b6a0930060201000201000180380100000000001976a9141cabd296e753837c086da7a45a6c2fe0d49d7b7b88ac00000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('OP_RETURN 300602010002010001', rpc_result['vout'][0]['scriptPubKey']['asm']) # verify that we have not altered scriptPubKey processing even of a specially crafted P2PKH pubkeyhash and P2SH redeem script hash that is made to pass the der signature checks tx = '01000000018d1f5635abd06e2c7e2ddf58dc85b3de111e4ad6e0ab51bb0dcf5e84126d927300000000fdfe0000483045022100ae3b4e589dfc9d48cb82d41008dc5fa6a86f94d5c54f9935531924602730ab8002202f88cf464414c4ed9fa11b773c5ee944f66e9b05cc1e51d97abc22ce098937ea01483045022100b44883be035600e9328a01b66c7d8439b74db64187e76b99a68f7893b701d5380220225bf286493e4c4adcf928c40f785422572eb232f84a0b83b0dea823c3a19c75014c695221020743d44be989540d27b1b4bbbcfd17721c337cb6bc9af20eb8a32520b393532f2102c0120a1dda9e51a938d39ddd9fe0ebc45ea97e1d27a7cbd671d5431416d3dd87210213820eb3d5f509d7438c9eeecb4157b2f595105e7cd564b3cdbb9ead3da41eed53aeffffffff02611e0000000000001976a914301102070101010101010102060101010101010188acee2a02000000000017a91430110207010101010101010206010101010101018700000000' rpc_result = self.nodes[0].decoderawtransaction(tx) assert_equal('OP_DUP OP_HASH160 3011020701010101010101020601010101010101 OP_EQUALVERIFY OP_CHECKSIG', rpc_result['vout'][0]['scriptPubKey']['asm']) assert_equal('OP_HASH160 3011020701010101010101020601010101010101 OP_EQUAL', rpc_result['vout'][1]['scriptPubKey']['asm']) # some more full transaction tests of varying specific scriptSigs. used instead of # tests in decodescript_script_sig because the decodescript RPC is specifically # for working on scriptPubKeys (argh!). push_signature = bytes_to_hex_str(txSave.vin[0].scriptSig)[2:(0x48*2+4)] signature = push_signature[2:] der_signature = signature[:-2] signature_sighash_decoded = der_signature + '[ALL]' signature_2 = der_signature + '82' push_signature_2 = '48' + signature_2 signature_2_sighash_decoded = der_signature + '[NONE|ANYONECANPAY]' # 1) P2PK scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature) rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal(signature_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # make sure that the sighash decodes come out correctly for a more complex / lesser used case. txSave.vin[0].scriptSig = hex_str_to_bytes(push_signature_2) rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal(signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 2) multisig scriptSig txSave.vin[0].scriptSig = hex_str_to_bytes('00' + push_signature + push_signature_2) rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal('0 ' + signature_sighash_decoded + ' ' + signature_2_sighash_decoded, rpc_result['vin'][0]['scriptSig']['asm']) # 3) test a scriptSig that contains more than push operations. # in fact, it contains an OP_RETURN with data specially crafted to cause improper decode if the code does not catch it. txSave.vin[0].scriptSig = hex_str_to_bytes('6a143011020701010101010101020601010101010101') rpc_result = self.nodes[0].decoderawtransaction(bytes_to_hex_str(txSave.serialize())) assert_equal('OP_RETURN 3011020701010101010101020601010101010101', rpc_result['vin'][0]['scriptSig']['asm'])
def serve_pil_image(pil_img): img_io = BytesIO() pil_img.save(img_io, 'PNG', quality=70) img_io.seek(0) return send_file(img_io, mimetype='image/png')
def write(self) -> bytes: data = BytesIO() data.write(Int(self.ID, False)) # No flags data.write(self.peer.write()) data.write(Int(self.expires)) data.write(Int(self.distance)) return data.getvalue()
def test_real(bytes_real, real): reader = PDFObjectReader(BytesIO(bytes_real)) result = reader.next_item() assert isinstance(result, cos.Real) and result == real
def _process(self): pdf = ProgrammeToPDF(self.event) return send_file('program.pdf', BytesIO(pdf.getPDFBin()), 'application/pdf')
def test_integer(bytes_integer, integer): reader = PDFObjectReader(BytesIO(bytes_integer)) result = reader.next_item() assert isinstance(result, cos.Integer) and result == integer
def test_name(bytes_name, unicode_name): reader = PDFObjectReader(BytesIO(bytes_name)) result = reader.next_item() assert isinstance(result, cos.Name) and str(result) == unicode_name