def test_create(self):
#mock file-like objects to read and write
mock_output = BytesIO()
test_bytes = b"Testing file!"
test_bytes_length = len(test_bytes)
mock_input = BytesIO(test_bytes)
create(output_file=mock_output,
input_file=mock_input,
input_file_length=test_bytes_length,
input_file_name="Testing.file",
piece_length=512000,
announce_url="http://localhost:8000",
comment="Test torrent file",
created_by="Martin Ashby's Test script...")
mock_output.seek(0)
output_dict = bdecode_file(mock_output)
#check the output has the appropriate keys.
info_dict = output_dict.get(b"info")
self.assertTrue(len(info_dict))
self.assertTrue(info_dict.get(b"piece length") == 512000)
self.assertTrue(info_dict.get(b"name") == b"Testing.file")
self.assertTrue(info_dict.get(b"length") == test_bytes_length)
self.assertTrue(info_dict.get(b"pieces"))
self.assertTrue(output_dict.get(b"announce") == b"http://localhost:8000")
self.assertTrue(type(output_dict.get(b"creation date")) == int)
self.assertTrue(output_dict.get(b"comment") == b"Test torrent file")
self.assertTrue(output_dict.get(b"created by") == b"Martin Ashby's Test script...")
def zip_layer_folder(dir_path, layer_name):
"""
Create a zip archive with the content of the folder located at `dir_path`
and name it with `layer_name`.
Parameters
----------
dir_path: str
The path to the temporary folder in which are located the files to
be zipped.
layer_name: str
The name of the concerned layer (will be used as file name for the
zip archive).
Returns
-------
raw_content: str
The zip archive
archive_name: str
The name of the archive (used later in the header of the response).
"""
filenames = os.listdir(dir_path)
zip_stream = BytesIO()
myZip = ZipFile(zip_stream, "w", compression=ZIP_DEFLATED)
for filename in filenames:
if not filename.endswith('.geojson'):
f_name = path_join(dir_path, filename)
myZip.write(f_name, filename, ZIP_DEFLATED)
myZip.close()
zip_stream.seek(0)
return zip_stream.read(), ''.join([layer_name, ".zip"])
def bytes_read_csv(b, header, kwargs, dtypes=None, columns=None):
""" Convert a block of bytes to a Pandas DataFrame
Parameters
----------
b: bytestring
The content to be parsed with pandas.read_csv
header: bytestring
An optional header to prepend to b
kwargs: dict
A dictionary of keyword arguments to be passed to pandas.read_csv
dtypes: dict
DTypes to assign to columns
See Also:
dask.dataframe.csv.read_csv_from_bytes
"""
bio = BytesIO()
if not b.startswith(header.rstrip()):
bio.write(header)
bio.write(b)
bio.seek(0)
df = pd.read_csv(bio, **kwargs)
if dtypes:
coerce_dtypes(df, dtypes)
if columns and (list(df.columns) != list(columns)):
raise ValueError("Columns do not match", df.columns, columns)
return df
def test_read_opts():
# tests if read is seeing option sets, at initialization and after
# initialization
arr = np.arange(6).reshape(1,6)
stream = BytesIO()
savemat(stream, {'a': arr})
rdr = MatFile5Reader(stream)
back_dict = rdr.get_variables()
rarr = back_dict['a']
assert_array_equal(rarr, arr)
rdr = MatFile5Reader(stream, squeeze_me=True)
assert_array_equal(rdr.get_variables()['a'], arr.reshape((6,)))
rdr.squeeze_me = False
assert_array_equal(rarr, arr)
rdr = MatFile5Reader(stream, byte_order=boc.native_code)
assert_array_equal(rdr.get_variables()['a'], arr)
# inverted byte code leads to error on read because of swapped
# header etc
rdr = MatFile5Reader(stream, byte_order=boc.swapped_code)
assert_raises(Exception, rdr.get_variables)
rdr.byte_order = boc.native_code
assert_array_equal(rdr.get_variables()['a'], arr)
arr = np.array(['a string'])
stream.truncate(0)
stream.seek(0)
savemat(stream, {'a': arr})
rdr = MatFile5Reader(stream)
assert_array_equal(rdr.get_variables()['a'], arr)
rdr = MatFile5Reader(stream, chars_as_strings=False)
carr = np.atleast_2d(np.array(list(arr.item()), dtype='U1'))
assert_array_equal(rdr.get_variables()['a'], carr)
rdr.chars_as_strings = True
assert_array_equal(rdr.get_variables()['a'], arr)
def test_str_round():
# from report by Angus McMorland on mailing list 3 May 2010
stream = BytesIO()
in_arr = np.array(['Hello', 'Foob'])
out_arr = np.array(['Hello', 'Foob '])
savemat(stream, dict(a=in_arr))
res = loadmat(stream)
# resulted in ['HloolFoa', 'elWrdobr']
assert_array_equal(res['a'], out_arr)
stream.truncate(0)
stream.seek(0)
# Make Fortran ordered version of string
in_str = in_arr.tostring(order='F')
in_from_str = np.ndarray(shape=a.shape,
dtype=in_arr.dtype,
order='F',
buffer=in_str)
savemat(stream, dict(a=in_from_str))
assert_array_equal(res['a'], out_arr)
# unicode save did lead to buffer too small error
stream.truncate(0)
stream.seek(0)
in_arr_u = in_arr.astype('U')
out_arr_u = out_arr.astype('U')
savemat(stream, {'a': in_arr_u})
res = loadmat(stream)
assert_array_equal(res['a'], out_arr_u)
def test_empty_string():
# make sure reading empty string does not raise error
estring_fname = pjoin(test_data_path, 'single_empty_string.mat')
fp = open(estring_fname, 'rb')
rdr = MatFile5Reader(fp)
d = rdr.get_variables()
fp.close()
assert_array_equal(d['a'], np.array([], dtype='U1'))
# empty string round trip. Matlab cannot distiguish
# between a string array that is empty, and a string array
# containing a single empty string, because it stores strings as
# arrays of char. There is no way of having an array of char that
# is not empty, but contains an empty string.
stream = BytesIO()
savemat(stream, {'a': np.array([''])})
rdr = MatFile5Reader(stream)
d = rdr.get_variables()
assert_array_equal(d['a'], np.array([], dtype='U1'))
stream.truncate(0)
stream.seek(0)
savemat(stream, {'a': np.array([], dtype='U1')})
rdr = MatFile5Reader(stream)
d = rdr.get_variables()
assert_array_equal(d['a'], np.array([], dtype='U1'))
stream.close()
def send_tryton_url(self, path):
self.send_response(300)
hostname = (config.get('jsonrpc', 'hostname')
or unicode(socket.getfqdn(), 'utf8'))
hostname = '.'.join(encodings.idna.ToASCII(part) for part in
hostname.split('.'))
values = {
'hostname': hostname,
'path': path,
}
content = BytesIO()
def write(str_):
content.write(str_.encode('utf-8'))
write('<html')
write('<head>')
write('<meta http-equiv="Refresh" '
'content="0;url=tryton://%(hostname)s%(path)s"/>' % values)
write('<title>Moved</title>')
write('</head>')
write('<body>')
write('<h1>Moved</h1>')
write('<p>This page has moved to '
'<a href="tryton://%(hostname)s%(path)s">'
'tryton://%(hostname)s%(path)s</a>.</p>' % values)
write('</body>')
write('</html>')
length = content.tell()
content.seek(0)
self.send_header('Location', 'tryton://%(hostname)s%(path)s' % values)
self.send_header('Content-type', 'text/html')
self.send_header('Content-Length', str(length))
self.end_headers()
self.copyfile(content, self.wfile)
content.close()
def test_save_dict():
# Test that dict can be saved (as recarray), loaded as matstruct
dict_types = ((dict, False),)
try:
from collections import OrderedDict
except ImportError:
pass
else:
dict_types += ((OrderedDict, True),)
ab_exp = np.array([[(1, 2)]], dtype=[('a', object), ('b', object)])
ba_exp = np.array([[(2, 1)]], dtype=[('b', object), ('a', object)])
for dict_type, is_ordered in dict_types:
# Initialize with tuples to keep order for OrderedDict
d = dict_type([('a', 1), ('b', 2)])
stream = BytesIO()
savemat(stream, {'dict': d})
stream.seek(0)
vals = loadmat(stream)['dict']
assert_equal(set(vals.dtype.names), set(['a', 'b']))
if is_ordered: # Input was ordered, output in ab order
assert_array_equal(vals, ab_exp)
else: # Not ordered input, either order output
if vals.dtype.names[0] == 'a':
assert_array_equal(vals, ab_exp)
else:
assert_array_equal(vals, ba_exp)
def export(request):
path = request.GET.get("path")
if not path:
raise Http404
stores = Store.objects.live().filter(pootle_path__startswith=path)
num_items = stores.count()
if not num_items:
raise Http404
if num_items == 1:
store = stores.get()
contents = BytesIO(store.serialize())
name = os.path.basename(store.pootle_path)
contents.seek(0)
return download(contents.read(), name, "application/octet-stream")
# zip all the stores together
f = BytesIO()
prefix = path.strip("/").replace("/", "-")
if not prefix:
prefix = "export"
with BytesIO() as f:
with ZipFile(f, "w") as zf:
for store in stores:
try:
data = store.serialize()
except Exception as e:
logging.error("Could not serialize %r: %s",
store.pootle_path, e)
continue
zf.writestr(prefix + store.pootle_path, data)
return download(f.getvalue(), "%s.zip" % (prefix), "application/zip")
def open(self, name, mode='rb'):
resp = self.b2.download_file(name)
output = BytesIO()
output.write(resp)
output.seek(0)
return File(output, name)
class TestFileUploadParser(TestCase):
def setUp(self):
class MockRequest(object):
pass
from io import BytesIO
self.stream = BytesIO(
"Test text file".encode('utf-8')
)
request = MockRequest()
request.upload_handlers = (MemoryFileUploadHandler(),)
request.META = {
'HTTP_CONTENT_DISPOSITION': 'Content-Disposition: inline; filename=file.txt'.encode('utf-8'),
'HTTP_CONTENT_LENGTH': 14,
}
self.parser_context = {'request': request, 'kwargs': {}}
def test_parse(self):
""" Make sure the `QueryDict` works OK """
parser = FileUploadParser()
self.stream.seek(0)
data_and_files = parser.parse(self.stream, None, self.parser_context)
file_obj = data_and_files.files['file']
self.assertEqual(file_obj._size, 14)
def test_get_filename(self):
parser = FileUploadParser()
filename = parser.get_filename(self.stream, None, self.parser_context)
self.assertEqual(filename, 'file.txt'.encode('utf-8'))
def __init__(self, strings=()):
self.strings = OrderedDict((s, 0) for s in strings)
self.records = []
offset = 0
buf = BytesIO()
for key in tuple(self.strings.iterkeys()):
utf8 = utf8_text(key[:self.MAX_STRING_LENGTH])
l = len(utf8)
sz_bytes = encint(l)
raw = sz_bytes + utf8
if 0xfbf8 - buf.tell() < 6 + len(raw):
# Records in PDB files cannot be larger than 0x10000, so we
# stop well before that.
pad = 0xfbf8 - buf.tell()
buf.write(b'\0' * pad)
self.records.append(buf.getvalue())
buf.seek(0), buf.truncate(0)
offset = len(self.records) * 0x10000
buf.write(raw)
self.strings[key] = offset
offset += len(raw)
val = buf.getvalue()
if val:
self.records.append(align_block(val))
def deserialize(schema, binary):
bytes_writer = BytesIO()
bytes_writer.write(binary)
bytes_writer.seek(0)
res = fastavro.schemaless_reader(bytes_writer, schema)
return res
class FakePayload(object):
"""
A wrapper around BytesIO that restricts what can be read since data from
the network can't be seeked and cannot be read outside of its content
length. This makes sure that views can't do anything under the test client
that wouldn't work in Real Life.
"""
def __init__(self, content=None):
self.__content = BytesIO()
self.__len = 0
self.read_started = False
if content is not None:
self.write(content)
def __len__(self):
return self.__len
def read(self, num_bytes=None):
if not self.read_started:
self.__content.seek(0)
self.read_started = True
if num_bytes is None:
num_bytes = self.__len or 0
assert self.__len >= num_bytes, "Cannot read more than the available bytes from the HTTP incoming data."
content = self.__content.read(num_bytes)
self.__len -= num_bytes
return content
def write(self, content):
if self.read_started:
raise ValueError("Unable to write a payload after he's been read")
content = force_bytes(content)
self.__content.write(content)
self.__len += len(content)
def _save(im, fp, filename):
fp.write(_MAGIC) # (2+2)
sizes = im.encoderinfo.get("sizes",
[(16, 16), (24, 24), (32, 32), (48, 48),
(64, 64), (128, 128), (255, 255)])
width, height = im.size
filter(lambda x: False if (x[0] > width or x[1] > height or
x[0] > 255 or x[1] > 255) else True, sizes)
sizes = sorted(sizes, key=lambda x: x[0])
fp.write(struct.pack("H", len(sizes))) # idCount(2)
offset = fp.tell() + len(sizes)*16
for size in sizes:
width, height = size
fp.write(struct.pack("B", width)) # bWidth(1)
fp.write(struct.pack("B", height)) # bHeight(1)
fp.write(b"\0") # bColorCount(1)
fp.write(b"\0") # bReserved(1)
fp.write(b"\0\0") # wPlanes(2)
fp.write(struct.pack("H", 32)) # wBitCount(2)
image_io = BytesIO()
tmp = im.copy()
tmp.thumbnail(size, Image.ANTIALIAS)
tmp.save(image_io, "png")
image_io.seek(0)
image_bytes = image_io.read()
bytes_len = len(image_bytes)
fp.write(struct.pack("I", bytes_len)) # dwBytesInRes(4)
fp.write(struct.pack("I", offset)) # dwImageOffset(4)
current = fp.tell()
fp.seek(offset)
fp.write(image_bytes)
offset = offset + bytes_len
fp.seek(current)
class ZippedStoreShaWriter(Sha1Writer):
"""Remembers everything someone writes to it and generates a sha"""
__slots__ = ('buf', 'zip')
def __init__(self):
Sha1Writer.__init__(self)
self.buf = BytesIO()
self.zip = zlib.compressobj(zlib.Z_BEST_SPEED)
def __getattr__(self, attr):
return getattr(self.buf, attr)
def write(self, data):
alen = Sha1Writer.write(self, data)
self.buf.write(self.zip.compress(data))
return alen
def close(self):
self.buf.write(self.zip.flush())
def seek(self, offset, whence=getattr(os, 'SEEK_SET', 0)):
"""Seeking currently only supports to rewind written data
Multiple writes are not supported"""
if offset != 0 or whence != getattr(os, 'SEEK_SET', 0):
raise ValueError("Can only seek to position 0")
# END handle offset
self.buf.seek(0)
def getvalue(self):
""":return: string value from the current stream position to the end"""
return self.buf.getvalue()
def to_pptx(self):
logger.info('Converting svg -> html -> png -> pptx')
content = None
try:
# convert to png
png_fn = self._rasterize_png()
# create blank presentation slide layout
pres = Presentation()
blank_slidelayout = pres.slide_layouts[6]
slide = pres.slides.add_slide(blank_slidelayout)
self._pptx_add_title(slide)
self._pptx_add_url(slide)
self._pptx_add_png(slide, png_fn)
self._pptx_add_hawc_logo(slide)
# save as object
content = BytesIO()
pres.save(content)
content.seek(0)
except Exception as e:
logger.error(e, exc_info=True)
finally:
self.cleanup()
return content
class ChunkBuffer:
BUFFER_SIZE = 1 * 1024 * 1024
def __init__(self, key, chunker_params=CHUNKER_PARAMS):
self.buffer = BytesIO()
self.packer = msgpack.Packer(unicode_errors='surrogateescape')
self.chunks = []
self.key = key
self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
def add(self, item):
self.buffer.write(self.packer.pack(StableDict(item)))
if self.is_full():
self.flush()
def write_chunk(self, chunk):
raise NotImplementedError
def flush(self, flush=False):
if self.buffer.tell() == 0:
return
self.buffer.seek(0)
chunks = list(bytes(s) for s in self.chunker.chunkify(self.buffer))
self.buffer.seek(0)
self.buffer.truncate(0)
# Leave the last partial chunk in the buffer unless flush is True
end = None if flush or len(chunks) == 1 else -1
for chunk in chunks[:end]:
self.chunks.append(self.write_chunk(chunk))
if end == -1:
self.buffer.write(chunks[-1])
def is_full(self):
return self.buffer.tell() > self.BUFFER_SIZE
def test_simple_bytesio(self):
f = BytesIO()
c = Commit()
c.committer = c.author = b"Jelmer <*****@*****.**>"
c.commit_time = c.author_time = 1271350201
c.commit_timezone = c.author_timezone = 0
c.message = b"This is the first line\nAnd this is the second line.\n"
c.tree = Tree().id
write_commit_patch(f, c, b"CONTENTS", (1, 1), version="custom")
f.seek(0)
lines = f.readlines()
self.assertTrue(lines[0].startswith(b"From 0b0d34d1b5b596c928adc9a727a4b9e03d025298"))
self.assertEqual(lines[1], b"From: Jelmer <*****@*****.**>\n")
self.assertTrue(lines[2].startswith(b"Date: "))
self.assertEqual(
[
b"Subject: [PATCH 1/1] This is the first line\n",
b"And this is the second line.\n",
b"\n",
b"\n",
b"---\n",
],
lines[3:8],
)
self.assertEqual([b"CONTENTS-- \n", b"custom\n"], lines[-2:])
if len(lines) >= 12:
# diffstat may not be present
self.assertEqual(lines[8], b" 0 files changed\n")
def test_commit_serialization(self):
assert_commit_serialization(self.gitrwrepo, self.gitrwrepo.head, True)
rwrepo = self.gitrwrepo
make_object = rwrepo.odb.store
# direct serialization - deserialization can be tested afterwards
# serialization is probably limited on IO
hc = rwrepo.commit(rwrepo.head)
nc = 5000
st = time()
for i in xrange(nc):
cm = Commit(rwrepo, Commit.NULL_BIN_SHA, hc.tree,
hc.author, hc.authored_date, hc.author_tz_offset,
hc.committer, hc.committed_date, hc.committer_tz_offset,
str(i), parents=hc.parents, encoding=hc.encoding)
stream = BytesIO()
cm._serialize(stream)
slen = stream.tell()
stream.seek(0)
cm.binsha = make_object(IStream(Commit.type, slen, stream)).binsha
# END commit creation
elapsed = time() - st
print("Serialized %i commits to loose objects in %f s ( %f commits / s )"
% (nc, elapsed, nc / elapsed), file=sys.stderr)
def create_dummy_image():
file = BytesIO()
image = Image.new('RGBA', size=(50, 50), color=(155, 0, 0))
image.save(file, 'png')
file.name = 'test_image.png'
file.seek(0)
return file
def _process(self):
f = request.files['logo']
try:
img = Image.open(f)
except IOError:
flash(_('You cannot upload this file as a logo.'), 'error')
return jsonify_data(content=None)
if img.format.lower() not in {'jpeg', 'png', 'gif'}:
flash(_('The file has an invalid format ({format})').format(format=img.format), 'error')
return jsonify_data(content=None)
if img.mode == 'CMYK':
flash(_('The logo you uploaded is using the CMYK colorspace and has been converted to RGB. Please check if '
'the colors are correct and convert it manually if necessary.'), 'warning')
img = img.convert('RGB')
image_bytes = BytesIO()
img.save(image_bytes, 'PNG')
image_bytes.seek(0)
content = image_bytes.read()
self.event.logo = content
self.event.logo_metadata = {
'hash': crc32(content),
'size': len(content),
'filename': os.path.splitext(secure_filename(f.filename, 'logo'))[0] + '.png',
'content_type': 'image/png'
}
flash(_('New logo saved'), 'success')
logger.info("New logo '%s' uploaded by %s (%s)", f.filename, session.user, self.event)
return jsonify_data(content=get_logo_data(self.event))
def upload(self, path, filename,buffersize=None,callback=None, local_path=None):
try:
self.error = None
#print('Upload = ' + path + filename)
#print('Size = %.1f kB' % (os.path.getsize(filename) / 1024.0))
#print('start upload')
if local_path:
file_obj = open(local_path+filename, 'rb')
else:
file_obj = open(filename, 'rb')
offset = 0
while True:
if not buffersize:
filesize = self.conn.storeFile(self.service_name, path+filename, file_obj)
break
else:
buffer_obj = file_obj.read(buffersize)
if buffer_obj:
buffer_fileobj = BytesIO()
buffer_fileobj.write(buffer_obj)
buffer_fileobj.seek(0)
offset_new = self.conn.storeFileFromOffset(self.service_name, path+filename, buffer_fileobj, offset=offset, truncate=False)
#return the file position where the next byte will be written.
offset = offset_new
if callback:
callback(offset)
else:
break
file_obj.close()
#print('upload finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def download(self, path, filename,buffersize=None,callback=None, local_path=None):
try:
self.error = None
#print('Download = ' + path + filename)
attr = self.conn.getAttributes(self.service_name, path+filename)
#print('Size = %.1f kB' % (attr.file_size / 1024.0))
#print('start download')
file_obj = BytesIO()
if local_path:
fw = open(local_path+filename, 'wb')
else:
fw = open(filename, 'wb')
offset = 0
transmit =0
while True:
if not buffersize:
file_attributes, filesize = self.conn.retrieveFile(self.service_name, path+filename, file_obj)
else:
file_attributes, filesize = self.conn.retrieveFileFromOffset(self.service_name, path+filename, file_obj,offset=offset,max_length=buffersize)
if callback:
transmit = transmit + filesize
callback(transmit)
file_obj.seek(offset)
for line in file_obj:
fw.write(line)
offset = offset + filesize
if (not buffersize) or (filesize == 0):
break
fw.close()
#print('download finished')
except Exception as e:
if self.print_errors:
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
else:
self.error = 'Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + str(type(e).__name__) + str(e)
def test_read_atom(self):
"""Test that all fields are read from PDB ATOM records"""
s = BytesIO()
# PDB is fixed-format; we should be able to read coordinates even
# without spaces between them
s.write(b'ATOM 1 N ALA A 5 3000.0001000.4002000.600'
b' 2.00 6.40 N\n')
s.seek(0)
m = IMP.Model()
pdb = IMP.atom.read_pdb(s, m)
atoms = IMP.atom.get_by_type(pdb, IMP.atom.ATOM_TYPE)
self.assertEqual(len(atoms), 1)
a = IMP.atom.Atom(atoms[0])
r = IMP.atom.Residue(a.get_parent())
c = IMP.atom.Chain(r.get_parent())
self.assertEqual(a.get_input_index(), 1)
self.assertEqual(a.get_atom_type().get_string(), 'N')
# Note: currently don't read alternate location or insertion code
self.assertEqual(r.get_residue_type().get_string(), 'ALA')
self.assertEqual(c.get_id(), 'A')
self.assertEqual(r.get_index(), 5)
coord = IMP.core.XYZ(a).get_coordinates()
self.assertAlmostEqual(coord[0], 3000.000, delta=0.001)
self.assertAlmostEqual(coord[1], 1000.400, delta=0.001)
self.assertAlmostEqual(coord[2], 2000.600, delta=0.001)
self.assertAlmostEqual(a.get_occupancy(), 2.00, delta=0.01)
self.assertAlmostEqual(a.get_temperature_factor(), 6.40, delta=0.01)
def write_data(self, item, last=False, stream_id=None):
"""
Send a DATA frame that is tracked by the local state machine.
Write a DATA frame using the H2 Connection object, will only work if the stream is in a state to send
DATA frames. Uses flow control to split data into multiple data frames if it exceeds the size that can
be in a single frame.
:param item: The content of the DATA frame
:param last: Flag to signal if this is the last frame in stream.
:param stream_id: Id of stream to send frame on. Will use the request stream ID if None
"""
if isinstance(item, (text_type, binary_type)):
data = BytesIO(self.encode(item))
else:
data = item
# Find the length of the data
data.seek(0, 2)
data_len = data.tell()
data.seek(0)
# If the data is longer than max payload size, need to write it in chunks
payload_size = self.get_max_payload_size()
while data_len > payload_size:
self.write_data_frame(data.read(payload_size), False, stream_id)
data_len -= payload_size
payload_size = self.get_max_payload_size()
self.write_data_frame(data.read(), last, stream_id)
def _download_part( self, part_num ):
"""
Download a part from the source URL. Returns a BytesIO buffer. The buffer's tell() method
will return the size of the downloaded part, which may be less than the requested part
size if the part is the last one for the URL.
"""
buf = BytesIO( )
with closing( pycurl.Curl( ) ) as c:
c.setopt( c.URL, self.url )
c.setopt( c.WRITEDATA, buf )
c.setopt( c.FAILONERROR, 1 )
start, end = self._get_part_range( part_num )
c.setopt( c.RANGE, "%i-%i" % (start, end - 1) )
try:
c.perform( )
except pycurl.error as e:
error_code, message = e
if error_code == c.E_BAD_DOWNLOAD_RESUME: # bad range for FTP
pass
elif error_code == c.E_HTTP_RETURNED_ERROR:
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4.17
if c.getinfo( c.RESPONSE_CODE ) == 416:
pass
else:
raise
else:
raise
buf.truncate( 0 )
buf.seek( 0 )
return buf
def tar_stream(repo, tree, mtime, format=''):
"""
Returns a generator that lazily assembles a .tar.gz archive, yielding it in
pieces (bytestrings). To obtain the complete .tar.gz binary file, simply
concatenate these chunks.
'repo' and 'tree' are the dulwich Repo and Tree objects the archive shall be
created from. 'mtime' is a UNIX timestamp that is assigned as the modification
time of all files in the resulting .tar.gz archive.
"""
buf = BytesIO()
with closing(tarfile.open(None, "w:%s" % format, buf)) as tar:
for entry_abspath, entry in walk_tree(repo, tree):
try:
blob = repo[entry.sha]
except KeyError:
# Entry probably refers to a submodule, which we don't yet support.
continue
data = ListBytesIO(blob.chunked)
info = tarfile.TarInfo()
info.name = entry_abspath
info.size = blob.raw_length()
info.mode = entry.mode
info.mtime = mtime
tar.addfile(info, data)
yield buf.getvalue()
buf.truncate(0)
buf.seek(0)
yield buf.getvalue()
def get(self, path, mode='rb'):
name = self.get_name(path)
key = self.bucket.get_key(name)
f = BytesIO()
key.get_contents_to_file(f)
f.seek(0)
return f
def unbind(filedata):
"""This is old code I didn't bother refactoring, sorry for the mess"""
binded = BytesIO(filedata)
binded.seek(-4, os.SEEK_END)
indexsize = binded.read(4)
indexsize = struct.unpack('<I', indexsize)[0]
binded.seek(0 - (indexsize + 4), os.SEEK_END)
endofdata = binded.tell()
indexdata = binded.read(indexsize - 1)
binded.seek(endofdata, os.SEEK_SET)
files = []
for line in reversed(indexdata.splitlines()):
file = line.split(b'/')
if (file[0] != b"__END_OF_SERIES_OF_BINDED_FILES__"):
fstart = int(file[1], 10)
fend = binded.tell()
fsize = (fend - fstart)
binded.seek(-fsize, os.SEEK_CUR)
buffer = binded.read(fsize + 1)
name = file[0].decode('utf-8') + getfileext(buffer)
if (fstart > 0):
binded.seek(fstart - 1, os.SEEK_SET)
files.append((name, buffer))
return files
async def draw_herostatstable(table_args, hero_stat_categories,
leveled_hero_stats):
category = None
for cat in hero_stat_categories:
if any(stat["stat"] == table_args.stat for stat in cat["stats"]):
category = cat
break
if category is None:
raise UserError("Couldn't find referenced stat")
stats = category["stats"]
# sort / get data
hero_data = leveled_hero_stats[table_args.hero_level]
hero_data = sorted(hero_data,
key=lambda hero: hero.get(table_args.stat),
reverse=not table_args.reverse)
hero_data = hero_data[0:table_args.hero_count]
table = Table(border_size=10)
stat_highlight_color = discord_color1
table_background = discord_color2
table_border_color = discord_color1
header_row = [TextCell("")]
for stat in stats:
header_row.append(
SlantedTextCell(stat["name"],
font_size=20,
background=stat_highlight_color if stat["stat"]
== table_args.stat else table_background,
border_color=table_border_color,
border_size=2,
rotation=45))
header_height = max(cell.height for cell in header_row)
padding_right = int(header_height / math.tan(header_row[-1].rotation_rad))
table.border_size[1] = padding_right
table.add_row(header_row)
i = 0
for hero in hero_data:
cell_background = table_background
#cell_background = stat_highlight_color if i % 2 else table_background
new_row = [
ImageCell(img=await get_hero_icon(hero.get("id")),
padding=3,
border_color=table_border_color,
border_size=2,
background=table_background)
]
for stat in stats:
value = hero.get(stat["stat"])
if stat.get("display") == "resistance_percentage":
value = 100 * (1 - value)
if stat.get("display") == "int":
value = round(value)
value = f"{value:.2f}"
value = re.sub("\.0+$", "", value)
if stat.get("display") == "resistance_percentage":
value += "%"
new_row.append(
TextCell(value,
font_size=16,
padding=10,
border_color=table_border_color,
border_size=2,
background=stat_highlight_color if stat["stat"]
== table_args.stat else cell_background))
table.add_row(new_row)
i += 1
image = table.render()
fp = BytesIO()
image.save(fp, format="PNG")
fp.seek(0)
return fp
def create_aww_performance_excel_file(excel_data,
data_type,
month,
state,
district=None,
block=None):
aggregation_level = 3 if block else (2 if district else 1)
export_info = excel_data[1][1]
excel_data = [line[aggregation_level:] for line in excel_data[0][1]]
thin_border = Border(left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin'))
warp_text_alignment = Alignment(wrap_text=True)
bold_font = Font(bold=True)
blue_fill = PatternFill("solid", fgColor="B3C5E5")
grey_fill = PatternFill("solid", fgColor="BFBFBF")
workbook = Workbook()
worksheet = workbook.active
worksheet.title = "AWW Performance Report"
worksheet.sheet_view.showGridLines = False
# sheet title
worksheet.merge_cells('B2:{0}2'.format("K" if aggregation_level == 3 else (
"L" if aggregation_level == 2 else "M")))
title_cell = worksheet['B2']
title_cell.fill = PatternFill("solid", fgColor="4472C4")
title_cell.value = "AWW Performance Report for the month of {}".format(
month)
title_cell.font = Font(size=18, color="FFFFFF")
title_cell.alignment = Alignment(horizontal="center")
# sheet header
header_cells = ["B3", "C3", "D3", "E3", "F3", "G3", "H3", "I3", "J3", "K3"]
if aggregation_level < 3:
header_cells.append("L3")
if aggregation_level < 2:
header_cells.append("M3")
for cell in header_cells:
worksheet[cell].fill = blue_fill
worksheet[cell].font = bold_font
worksheet[cell].alignment = warp_text_alignment
worksheet.merge_cells('B3:C3')
worksheet['B3'].value = "State: {}".format(state)
if district:
worksheet['D3'].value = "District: {}".format(district)
worksheet.merge_cells('E3:F3')
if block:
worksheet['E3'].value = "Block: {}".format(block)
# table header
table_header_position_row = 5
headers = ["S.No"]
if aggregation_level < 2:
headers.append("District")
if aggregation_level < 3:
headers.append("Block")
headers.extend([
'Supervisor', 'AWC', 'AWW Name', 'AWW Contact Number',
'Home Visits Conducted', 'Weighing Efficiency',
'AWW Eligible for Incentive', 'Number of Days AWC was Open',
'AWH Eligible for Incentive'
])
columns = ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K']
if aggregation_level < 3:
columns.append('L')
if aggregation_level < 2:
columns.append('M')
table_header = {}
for col, header in zip(columns, headers):
table_header[col] = header
for column, value in table_header.items():
cell = "{}{}".format(column, table_header_position_row)
worksheet[cell].fill = grey_fill
worksheet[cell].border = thin_border
worksheet[cell].font = bold_font
worksheet[cell].alignment = warp_text_alignment
worksheet[cell].value = value
# table contents
row_position = table_header_position_row + 1
for enum, row in enumerate(excel_data[1:], start=1):
for column_index in range(len(columns)):
column = columns[column_index]
cell = "{}{}".format(column, row_position)
worksheet[cell].border = thin_border
if column_index == 0:
worksheet[cell].value = enum
else:
worksheet[cell].value = row[column_index - 1]
row_position += 1
# sheet dimensions
title_row = worksheet.row_dimensions[2]
title_row.height = 23
worksheet.row_dimensions[table_header_position_row].height = 46
widths = {}
widths_columns = ['A']
widths_columns.extend(columns)
standard_widths = [4, 7, 15]
standard_widths.extend([15] * (3 - aggregation_level))
standard_widths.extend([13, 12, 13, 15, 11, 14, 14])
standard_widths.append(14)
for col, width in zip(widths_columns, standard_widths):
widths[col] = width
widths['C'] = max(widths['C'], len(state) * 4 // 3 if state else 0)
widths['D'] = 13 + (len(district) * 4 // 3 if district else 0)
widths['F'] = max(widths['F'], len(block) * 4 // 3 if block else 0)
for column in ["C", "E", "G"]:
if widths[column] > 25:
worksheet.row_dimensions[3].height = max(
16 * ((widths[column] // 25) + 1),
worksheet.row_dimensions[3].height)
widths[column] = 25
columns = columns[1:]
# column widths based on table contents
for column_index in range(len(columns)):
widths[columns[column_index]] = max(
widths[columns[column_index]],
max(
len(row[column_index].decode('utf-8') if isinstance(
row[column_index], bytes) else six.
text_type(row[column_index])) for row in excel_data[1:]) *
4 // 3 if len(excel_data) >= 2 else 0)
for column, width in widths.items():
worksheet.column_dimensions[column].width = width
# export info
worksheet2 = workbook.create_sheet("Export Info")
worksheet2.column_dimensions['A'].width = 14
for n, export_info_item in enumerate(export_info, start=1):
worksheet2['A{0}'.format(n)].value = export_info_item[0]
worksheet2['B{0}'.format(n)].value = export_info_item[1]
# saving file
key = get_performance_report_blob_key(state, district, block, month,
'xlsx')
export_file = BytesIO()
icds_file, _ = IcdsFile.objects.get_or_create(blob_id=key,
data_type=data_type)
workbook.save(export_file)
export_file.seek(0)
icds_file.store_file_in_blobdb(export_file, expired=None)
icds_file.save()
return key
for t in x:
r = np.random.normal(loc=0.0, scale=(0.5 + t * t / 3), size=None)
y.append(r)
return x, 1.726 * x - 0.84 + np.array(y)
x, y = make_random_data()
plt.plot(x, y, 'o')
#plt.savefig('images/14_03.png', dpi=300)
#plt.show()
img = BytesIO()
plt.savefig(img, dpi=300)
plt.close()
img.seek(0)
ml14_plot_url1 = b64encode(img.getvalue()).decode('ascii')
# ### Executing objects in a TensorFlow graph using their names
# In[24]:
## train/test splits:
x_train, y_train = x[:100], y[:100]
x_test, y_test = x[100:], y[100:]
## training the model
n_epochs = 500
training_costs = []
with tf.Session(graph=g) as sess:
def _generate_card(buffer: BytesIO, content: dict) -> BytesIO:
"""
Generate a card from snake information.
Written by juan and Someone during the first code jam.
"""
snake = Image.open(buffer)
# Get the size of the snake icon, configure the height of the image box (yes, it changes)
icon_width = 347 # Hardcoded, not much i can do about that
icon_height = int((icon_width / snake.width) * snake.height)
frame_copies = icon_height // CARD['frame'].height + 1
snake.thumbnail((icon_width, icon_height))
# Get the dimensions of the final image
main_height = icon_height + CARD['top'].height + CARD['bottom'].height
main_width = CARD['frame'].width
# Start creating the foreground
foreground = Image.new("RGBA", (main_width, main_height), (0, 0, 0, 0))
foreground.paste(CARD['top'], (0, 0))
# Generate the frame borders to the correct height
for offset in range(frame_copies):
position = (0, CARD['top'].height + offset * CARD['frame'].height)
foreground.paste(CARD['frame'], position)
# Add the image and bottom part of the image
foreground.paste(snake, (36, CARD['top'].height)) # Also hardcoded :(
foreground.paste(CARD['bottom'], (0, CARD['top'].height + icon_height))
# Setup the background
back = random.choice(CARD['backs'])
back_copies = main_height // back.height + 1
full_image = Image.new("RGBA", (main_width, main_height), (0, 0, 0, 0))
# Generate the tiled background
for offset in range(back_copies):
full_image.paste(back, (16, 16 + offset * back.height))
# Place the foreground onto the final image
full_image.paste(foreground, (0, 0), foreground)
# Get the first two sentences of the info
description = '.'.join(content['info'].split(".")[:2]) + '.'
# Setup positioning variables
margin = 36
offset = CARD['top'].height + icon_height + margin
# Create blank rectangle image which will be behind the text
rectangle = Image.new("RGBA", (main_width, main_height), (0, 0, 0, 0))
# Draw a semi-transparent rectangle on it
rect = ImageDraw.Draw(rectangle)
rect.rectangle(
(margin, offset, main_width - margin, main_height - margin),
fill=(63, 63, 63, 128))
# Paste it onto the final image
full_image.paste(rectangle, (0, 0), mask=rectangle)
# Draw the text onto the final image
draw = ImageDraw.Draw(full_image)
for line in textwrap.wrap(description, 36):
draw.text([margin + 4, offset], line, font=CARD['font'])
offset += CARD['font'].getsize(line)[1]
# Get the image contents as a BufferIO object
buffer = BytesIO()
full_image.save(buffer, 'PNG')
buffer.seek(0)
return buffer
def fig_to_base64(figure):
plt_bytes = BytesIO()
figure.savefig(plt_bytes, format='png')
plt_bytes.seek(0)
return plt_bytes
def run_test(self):
url = urlparse.urlparse(self.nodes[0].url)
print "Mining blocks..."
self.nodes[0].generate(1)
self.sync_all()
self.nodes[2].generate(100)
self.sync_all()
assert_equal(self.nodes[0].getbalance(), 500)
txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
self.sync_all()
self.nodes[2].generate(1)
self.sync_all()
bb_hash = self.nodes[0].getbestblockhash()
assert_equal(self.nodes[1].getbalance(),
Decimal("0.1")) #balance now should be 0.1 on node 1
# load the latest 0.1 tx over the REST API
json_string = http_get_call(
url.hostname, url.port,
'/rest/tx/' + txid + self.FORMAT_SEPARATOR + "json")
json_obj = json.loads(json_string)
vintx = json_obj['vin'][0][
'txid'] # get the vin to later check for utxo (should be spent by then)
# get n of 0.1 outpoint
n = 0
for vout in json_obj['vout']:
if vout['value'] == 0.1:
n = vout['n']
######################################
# GETUTXOS: query a unspent outpoint #
######################################
json_request = '/checkmempool/' + txid + '-' + str(n)
json_string = http_get_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
#check chainTip response
assert_equal(json_obj['chaintipHash'], bb_hash)
#make sure there is one utxo
assert_equal(len(json_obj['utxos']), 1)
assert_equal(json_obj['utxos'][0]['value'], 0.1)
################################################
# GETUTXOS: now query a already spent outpoint #
################################################
json_request = '/checkmempool/' + vintx + '-0'
json_string = http_get_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
#check chainTip response
assert_equal(json_obj['chaintipHash'], bb_hash)
#make sure there is no utox in the response because this oupoint has been spent
assert_equal(len(json_obj['utxos']), 0)
#check bitmap
assert_equal(json_obj['bitmap'], "0")
##################################################
# GETUTXOS: now check both with the same request #
##################################################
json_request = '/checkmempool/' + txid + '-' + str(
n) + '/' + vintx + '-0'
json_string = http_get_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
assert_equal(len(json_obj['utxos']), 1)
assert_equal(json_obj['bitmap'], "10")
#test binary response
bb_hash = self.nodes[0].getbestblockhash()
binaryRequest = b'\x01\x02'
binaryRequest += hex_str_to_bytes(txid)
binaryRequest += pack("i", n)
binaryRequest += hex_str_to_bytes(vintx)
binaryRequest += pack("i", 0)
bin_response = http_post_call(
url.hostname, url.port,
'/rest/getutxos' + self.FORMAT_SEPARATOR + 'bin', binaryRequest)
output = BytesIO()
output.write(bin_response)
output.seek(0)
chainHeight = unpack("i", output.read(4))[0]
hashFromBinResponse = hex(
deser_uint256(output))[2:].zfill(65).rstrip("L")
assert_equal(
bb_hash, hashFromBinResponse
) #check if getutxo's chaintip during calculation was fine
assert_equal(chainHeight, 102) #chain height must be 102
############################
# GETUTXOS: mempool checks #
############################
# do a tx and don't sync
txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
json_string = http_get_call(
url.hostname, url.port,
'/rest/tx/' + txid + self.FORMAT_SEPARATOR + "json")
json_obj = json.loads(json_string)
vintx = json_obj['vin'][0][
'txid'] # get the vin to later check for utxo (should be spent by then)
# get n of 0.1 outpoint
n = 0
for vout in json_obj['vout']:
if vout['value'] == 0.1:
n = vout['n']
json_request = '/' + txid + '-' + str(n)
json_string = http_get_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
assert_equal(
len(json_obj['utxos']), 0
) #there should be a outpoint because it has just added to the mempool
json_request = '/checkmempool/' + txid + '-' + str(n)
json_string = http_get_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
assert_equal(
len(json_obj['utxos']), 1
) #there should be a outpoint because it has just added to the mempool
#do some invalid requests
json_request = '{"checkmempool'
response = http_post_call(
url.hostname, url.port,
'/rest/getutxos' + self.FORMAT_SEPARATOR + 'json', json_request,
True)
assert_equal(
response.status,
500) #must be a 500 because we send a invalid json request
json_request = '{"checkmempool'
response = http_post_call(
url.hostname, url.port,
'/rest/getutxos' + self.FORMAT_SEPARATOR + 'bin', json_request,
True)
assert_equal(response.status,
500) #must be a 500 because we send a invalid bin request
response = http_post_call(
url.hostname, url.port,
'/rest/getutxos/checkmempool' + self.FORMAT_SEPARATOR + 'bin', '',
True)
assert_equal(response.status,
500) #must be a 500 because we send a invalid bin request
#test limits
json_request = '/checkmempool/'
for x in range(0, 20):
json_request += txid + '-' + str(n) + '/'
json_request = json_request.rstrip("/")
response = http_post_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json',
'', True)
assert_equal(response.status,
500) #must be a 500 because we exceeding the limits
json_request = '/checkmempool/'
for x in range(0, 15):
json_request += txid + '-' + str(n) + '/'
json_request = json_request.rstrip("/")
response = http_post_call(
url.hostname, url.port,
'/rest/getutxos' + json_request + self.FORMAT_SEPARATOR + 'json',
'', True)
assert_equal(response.status,
200) #must be a 500 because we exceeding the limits
self.nodes[0].generate(1) #generate block to not affect upcoming tests
self.sync_all()
################
# /rest/block/ #
################
# check binary format
response = http_get_call(
url.hostname, url.port,
'/rest/block/' + bb_hash + self.FORMAT_SEPARATOR + "bin", True)
assert_equal(response.status, 200)
assert_greater_than(int(response.getheader('content-length')), 80)
response_str = response.read()
# compare with block header
response_header = http_get_call(
url.hostname, url.port,
'/rest/headers/1/' + bb_hash + self.FORMAT_SEPARATOR + "bin", True)
assert_equal(response_header.status, 200)
assert_equal(int(response_header.getheader('content-length')), 80)
response_header_str = response_header.read()
assert_equal(response_str[0:80], response_header_str)
# check block hex format
response_hex = http_get_call(
url.hostname, url.port,
'/rest/block/' + bb_hash + self.FORMAT_SEPARATOR + "hex", True)
assert_equal(response_hex.status, 200)
assert_greater_than(int(response_hex.getheader('content-length')), 160)
response_hex_str = response_hex.read()
assert_equal(
encode(response_str, "hex_codec")[0:160], response_hex_str[0:160])
# compare with hex block header
response_header_hex = http_get_call(
url.hostname, url.port,
'/rest/headers/1/' + bb_hash + self.FORMAT_SEPARATOR + "hex", True)
assert_equal(response_header_hex.status, 200)
assert_greater_than(
int(response_header_hex.getheader('content-length')), 160)
response_header_hex_str = response_header_hex.read()
assert_equal(response_hex_str[0:160], response_header_hex_str[0:160])
assert_equal(
encode(response_header_str, "hex_codec")[0:160],
response_header_hex_str[0:160])
# check json format
block_json_string = http_get_call(
url.hostname, url.port,
'/rest/block/' + bb_hash + self.FORMAT_SEPARATOR + 'json')
block_json_obj = json.loads(block_json_string)
assert_equal(block_json_obj['hash'], bb_hash)
# compare with json block header
response_header_json = http_get_call(
url.hostname, url.port,
'/rest/headers/1/' + bb_hash + self.FORMAT_SEPARATOR + "json",
True)
assert_equal(response_header_json.status, 200)
response_header_json_str = response_header_json.read().decode('utf-8')
json_obj = json.loads(response_header_json_str, parse_float=Decimal)
assert_equal(len(json_obj),
1) #ensure that there is one header in the json response
assert_equal(json_obj[0]['hash'],
bb_hash) #request/response hash should be the same
#compare with normal RPC block response
rpc_block_json = self.nodes[0].getblock(bb_hash)
assert_equal(json_obj[0]['hash'], rpc_block_json['hash'])
assert_equal(json_obj[0]['confirmations'],
rpc_block_json['confirmations'])
assert_equal(json_obj[0]['height'], rpc_block_json['height'])
assert_equal(json_obj[0]['version'], rpc_block_json['version'])
assert_equal(json_obj[0]['merkleroot'], rpc_block_json['merkleroot'])
assert_equal(json_obj[0]['time'], rpc_block_json['time'])
assert_equal(json_obj[0]['nonce'], rpc_block_json['nonce'])
assert_equal(json_obj[0]['bits'], rpc_block_json['bits'])
assert_equal(json_obj[0]['difficulty'], rpc_block_json['difficulty'])
assert_equal(json_obj[0]['chainwork'], rpc_block_json['chainwork'])
assert_equal(json_obj[0]['previousblockhash'],
rpc_block_json['previousblockhash'])
#see if we can get 5 headers in one response
self.nodes[1].generate(5)
self.sync_all()
response_header_json = http_get_call(
url.hostname, url.port,
'/rest/headers/5/' + bb_hash + self.FORMAT_SEPARATOR + "json",
True)
assert_equal(response_header_json.status, 200)
response_header_json_str = response_header_json.read().decode('utf-8')
json_obj = json.loads(response_header_json_str)
assert_equal(len(json_obj), 5) #now we should have 5 header objects
# do tx test
tx_hash = block_json_obj['tx'][0]['txid']
json_string = http_get_call(
url.hostname, url.port,
'/rest/tx/' + tx_hash + self.FORMAT_SEPARATOR + "json")
json_obj = json.loads(json_string)
assert_equal(json_obj['txid'], tx_hash)
# check hex format response
hex_string = http_get_call(
url.hostname, url.port,
'/rest/tx/' + tx_hash + self.FORMAT_SEPARATOR + "hex", True)
assert_equal(hex_string.status, 200)
assert_greater_than(int(response.getheader('content-length')), 10)
# check block tx details
# let's make 3 tx and mine them on node 1
txs = []
txs.append(self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),
11))
txs.append(self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),
11))
txs.append(self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(),
11))
self.sync_all()
# check that there are exactly 3 transactions in the TX memory pool before generating the block
json_string = http_get_call(
url.hostname, url.port,
'/rest/mempool/info' + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
assert_equal(json_obj['size'], 3)
# the size of the memory pool should be greater than 3x ~100 bytes
assert_greater_than(json_obj['bytes'], 300)
# check that there are our submitted transactions in the TX memory pool
json_string = http_get_call(
url.hostname, url.port,
'/rest/mempool/contents' + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
for tx in txs:
assert_equal(tx in json_obj, True)
# now mine the transactions
newblockhash = self.nodes[1].generate(1)
self.sync_all()
#check if the 3 tx show up in the new block
json_string = http_get_call(
url.hostname, url.port,
'/rest/block/' + newblockhash[0] + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
for tx in json_obj['tx']:
if not 'coinbase' in tx['vin'][0]: #exclude coinbase
assert_equal(tx['txid'] in txs, True)
#check the same but without tx details
json_string = http_get_call(
url.hostname, url.port, '/rest/block/notxdetails/' +
newblockhash[0] + self.FORMAT_SEPARATOR + 'json')
json_obj = json.loads(json_string)
for tx in txs:
assert_equal(tx in json_obj['tx'], True)
#test rest bestblock
bb_hash = self.nodes[0].getbestblockhash()
json_string = http_get_call(url.hostname, url.port,
'/rest/chaininfo.json')
json_obj = json.loads(json_string)
assert_equal(json_obj['bestblockhash'], bb_hash)
def do_POST(self):
"""Serve a POST request."""
# First, we save the post data
r, info = self.deal_post_data()
print((r, info, "by: ", self.client_address))
# And write the response web page
f = BytesIO()
f.write(
b"<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\"><html>"
)
f.write(b"<title>qr-filetransfer</title>")
f.write(
b"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">"
)
f.write(
b"<link href=\"https://fonts.googleapis.com/css?family=Comfortaa\" rel=\"stylesheet\">"
)
f.write(
b"<link rel=\"icon\" href=\"https://raw.githubusercontent.com/sdushantha/qr-filetransfer/master/logo.png\" type=\"image/png\">"
)
f.write(b"<center>")
f.write(b"<body>")
f.write(
b"<h2 style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">Upload Result Page</h2>"
)
f.write(b"<hr>")
if r:
f.write(
b"<strong style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">Success: </strong>"
)
else:
f.write(
b"<strong style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">Failed: </strong>"
)
f.write((
"<span style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">%s</span><br>"
% info).encode())
f.write((
"<br><a href=\"%s\" style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">back</a>"
% self.headers['referer']).encode())
f.write(
b"<hr><small style=\"font-family: 'Comfortaa', cursive;color:'#263238';\">Powerd By: "
)
f.write(b"<a href=\"https://github.com/sdushantha/\">")
f.write(b"sdushantha</a> and \n")
f.write(b"<a href=\"https://github.com/npes87184/\">")
f.write(b"npes87184</a>, check new version at \n")
f.write(b"<a href=\"https://pypi.org/project/qr-filetransfer/\">")
f.write(b"here</a>.</small></body>\n</html>\n")
length = f.tell()
f.seek(0)
self.send_response(200)
self.send_header("Content-type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(length))
self.end_headers()
if f:
self.copyfile(f, self.wfile)
f.close()
def write_response(self, socket_file, data):
buf = BytesIO()
self._write(buf, data)
buf.seek(0)
socket_file.write(buf.getvalue())
socket_file.flush()
def read_partial(fileobj, stop_when=None, defer_size=None, force=False):
"""Parse a DICOM file until a condition is met.
Parameters
----------
fileobj : a file-like object
Note that the file will not close when the function returns.
stop_when :
Stop condition. See ``read_dataset`` for more info.
defer_size : int, str, None, optional
See ``read_file`` for parameter info.
force : boolean
See ``read_file`` for parameter info.
Notes
-----
Use ``read_file`` unless you need to stop on some condition
other than reaching pixel data.
Returns
-------
FileDataset instance or DicomDir instance.
See Also
--------
read_file
More generic file reading function.
"""
# Read preamble -- raise an exception if missing and force=False
preamble = read_preamble(fileobj, force)
file_meta_dataset = Dataset()
# Assume a transfer syntax, correct it as necessary
is_implicit_VR = True
is_little_endian = True
if preamble:
file_meta_dataset = _read_file_meta_info(fileobj)
transfer_syntax = file_meta_dataset.get("TransferSyntaxUID")
if transfer_syntax is None: # issue 258
pass
elif transfer_syntax == pydicom.uid.ImplicitVRLittleEndian:
pass
elif transfer_syntax == pydicom.uid.ExplicitVRLittleEndian:
is_implicit_VR = False
elif transfer_syntax == pydicom.uid.ExplicitVRBigEndian:
is_implicit_VR = False
is_little_endian = False
elif transfer_syntax == pydicom.uid.DeflatedExplicitVRLittleEndian:
# See PS3.6-2008 A.5 (p 71)
# when written, the entire dataset following
# the file metadata was prepared the normal way,
# then "deflate" compression applied.
# All that is needed here is to decompress and then
# use as normal in a file-like object
zipped = fileobj.read()
# -MAX_WBITS part is from comp.lang.python answer:
# groups.google.com/group/comp.lang.python/msg/e95b3b38a71e6799
unzipped = zlib.decompress(zipped, -zlib.MAX_WBITS)
fileobj = BytesIO(unzipped) # a file-like object
is_implicit_VR = False
else:
# Any other syntax should be Explicit VR Little Endian,
# e.g. all Encapsulated (JPEG etc) are ExplVR-LE
# by Standard PS 3.5-2008 A.4 (p63)
is_implicit_VR = False
else: # no header -- use the is_little_endian, implicit assumptions
file_meta_dataset.TransferSyntaxUID = pydicom.uid.ImplicitVRLittleEndian
endian_chr = "<"
element_struct = Struct(endian_chr + "HH2sH")
# Try reading first 8 bytes
group, elem, VR, length = element_struct.unpack(fileobj.read(8))
# Rewind file object
fileobj.seek(0)
# If the VR is a valid VR, assume Explicit VR transfer systax
from pydicom.values import converters
if not in_py2:
VR = VR.decode(default_encoding)
if VR in converters.keys():
is_implicit_VR = False
# Determine if group in low numbered range (Little vs Big Endian)
if group == 0: # got (0,0) group length. Not helpful.
# XX could use similar to http://www.dclunie.com/medical-image-faq/html/part2.html code example
msg = ("Not able to guess transfer syntax when first item "
"is group length")
raise NotImplementedError(msg)
if group < 2000:
file_meta_dataset.TransferSyntaxUID = pydicom.uid.ExplicitVRLittleEndian
else:
file_meta_dataset.TransferSyntaxUID = pydicom.uid.ExplicitVRBigEndian
is_little_endian = False
try:
dataset = read_dataset(fileobj,
is_implicit_VR,
is_little_endian,
stop_when=stop_when,
defer_size=defer_size)
except EOFError:
pass # error already logged in read_dataset
class_uid = file_meta_dataset.get("MediaStorageSOPClassUID", None)
if class_uid and class_uid == "Media Storage Directory Storage":
return DicomDir(fileobj, dataset, preamble, file_meta_dataset,
is_implicit_VR, is_little_endian)
else:
return FileDataset(fileobj, dataset, preamble, file_meta_dataset,
is_implicit_VR, is_little_endian)
def create_lady_supervisor_excel_file(excel_data, data_type, month,
aggregation_level):
export_info = excel_data[1][1]
state = export_info[1][1] if aggregation_level > 0 else ''
district = export_info[2][1] if aggregation_level > 1 else ''
block = export_info[3][1] if aggregation_level > 2 else ''
excel_data = [line[aggregation_level:] for line in excel_data[0][1]]
thin_border = Border(left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin'))
warp_text_alignment = Alignment(wrap_text=True)
bold_font = Font(bold=True)
blue_fill = PatternFill("solid", fgColor="B3C5E5")
grey_fill = PatternFill("solid", fgColor="BFBFBF")
workbook = Workbook()
worksheet = workbook.active
worksheet.title = "LS Performance Report"
worksheet.sheet_view.showGridLines = False
# sheet title
amount_of_columns = 9 - aggregation_level
last_column = string.ascii_uppercase[amount_of_columns]
worksheet.merge_cells('B2:{0}2'.format(last_column))
title_cell = worksheet['B2']
title_cell.fill = PatternFill("solid", fgColor="4472C4")
title_cell.value = "Lady Supervisor Performance Report for the {}".format(
month)
title_cell.font = Font(size=18, color="FFFFFF")
title_cell.alignment = Alignment(horizontal="center")
columns = [
string.ascii_uppercase[i] for i in range(1, amount_of_columns + 1)
]
# sheet header
header_cells = ['{0}3'.format(column) for column in columns]
for cell in header_cells:
worksheet[cell].fill = blue_fill
worksheet[cell].font = bold_font
worksheet[cell].alignment = warp_text_alignment
if state:
worksheet['B3'].value = "State: {}".format(state)
worksheet.merge_cells('B3:C3')
if district:
worksheet['D3'].value = "District: {}".format(district)
if block:
worksheet['E3'].value = "Block: {}".format(block)
date_cell = '{0}3'.format(last_column)
date_description_cell = '{0}3'.format(
string.ascii_uppercase[amount_of_columns - 1])
worksheet[date_description_cell].value = "Date when downloaded:"
worksheet[date_description_cell].alignment = Alignment(horizontal="right")
utc_now = datetime.now(pytz.utc)
now_in_india = utc_now.astimezone(india_timezone)
worksheet[date_cell].value = custom_strftime('{S} %b %Y', now_in_india)
worksheet[date_cell].alignment = Alignment(horizontal="right")
# table header
table_header_position_row = 5
header_data = excel_data[0]
headers = ["S.No"]
headers.extend(header_data)
table_header = {}
for col, header in zip(columns, headers):
table_header[col] = header
for column, value in table_header.items():
cell = "{}{}".format(column, table_header_position_row)
worksheet[cell].fill = grey_fill
worksheet[cell].border = thin_border
worksheet[cell].font = bold_font
worksheet[cell].alignment = warp_text_alignment
worksheet[cell].value = value
# table contents
row_position = table_header_position_row + 1
for enum, row in enumerate(excel_data[1:], start=1):
for column_index in range(len(columns)):
column = columns[column_index]
cell = "{}{}".format(column, row_position)
worksheet[cell].border = thin_border
if column_index == 0:
worksheet[cell].value = enum
else:
worksheet[cell].value = row[column_index - 1]
row_position += 1
# sheet dimensions
title_row = worksheet.row_dimensions[2]
title_row.height = 23
worksheet.row_dimensions[table_header_position_row].height = 46
widths = {}
widths_columns = ['A']
widths_columns.extend(columns)
standard_widths = [4, 7]
standard_widths.extend([15] * (4 - aggregation_level))
standard_widths.extend([25, 15, 25, 15])
for col, width in zip(widths_columns, standard_widths):
widths[col] = width
widths['C'] = max(widths['C'], len(state) * 4 // 3 if state else 0)
widths['D'] = 9 + (len(district) * 4 // 3 if district else 0)
widths['E'] = 8 + (len(block) * 4 // 3 if district else 0)
columns = columns[1:]
# column widths based on table contents
for column_index in range(len(columns)):
widths[columns[column_index]] = max(
widths[columns[column_index]],
max(
len(row[column_index].decode('utf-8') if isinstance(
row[column_index], bytes) else six.
text_type(row[column_index])) for row in excel_data[1:]) *
4 // 3 if len(excel_data) >= 2 else 0)
for column, width in widths.items():
worksheet.column_dimensions[column].width = width
# export info
worksheet2 = workbook.create_sheet("Export Info")
worksheet2.column_dimensions['A'].width = 14
for n, export_info_item in enumerate(export_info, start=1):
worksheet2['A{0}'.format(n)].value = export_info_item[0]
worksheet2['B{0}'.format(n)].value = export_info_item[1]
# saving file
file_hash = uuid.uuid4().hex
export_file = BytesIO()
icds_file = IcdsFile(blob_id=file_hash, data_type=data_type)
workbook.save(export_file)
export_file.seek(0)
icds_file.store_file_in_blobdb(export_file, expired=ONE_DAY)
icds_file.save()
return file_hash
def df_to_bytes_np_records_(df: pd.DataFrame) -> bytes:
b = BytesIO()
rec = df.to_records()
np.save(b, rec, allow_pickle=False)
b.seek(0)
return b.read()
def saveArray(array, key):
bio = BytesIO()
numpy.save(bio, array)
bio.flush()
bio.seek(0)
saveBytes(bio.getvalue(), key)
def saveImageToFile(image, filename):
print 'saves image ', filename
# save full image
image.save(filename)
#Main loop
while True:
# Restart shooting thread
if not liveViewThread.isAlive():
liveViewThread = CameraThread()
liveViewThread.start()
# View Loop
stream.seek(0)
for bug in camera.capture_continuous(stream,
format='jpeg',
use_video_port=True,
splitter_port=1):
t1 = time.time()
stream.seek(
0
) # "Rewind" the stream to the beginning so we can read its content
print('capture')
# take a picture
if GPIO.event_detected(SHOT_PIN):
liveViewThread.exit = True
image = Image.open(stream)
def df_to_bytes_parquet_(df: pd.DataFrame) -> bytes:
b = BytesIO()
table = pyarrow.Table.from_pandas(df)
parquet.write_table(table, b)
b.seek(0)
return b.read()
def export(self, **kwargs: Any) -> Response:
"""Export dashboards
---
get:
description: >-
Exports multiple Dashboards and downloads them as YAML files.
parameters:
- in: query
name: q
content:
application/json:
schema:
$ref: '#/components/schemas/get_export_ids_schema'
responses:
200:
description: Dashboard export
content:
text/plain:
schema:
type: string
400:
$ref: '#/components/responses/400'
401:
$ref: '#/components/responses/401'
404:
$ref: '#/components/responses/404'
422:
$ref: '#/components/responses/422'
500:
$ref: '#/components/responses/500'
"""
requested_ids = kwargs["rison"]
if is_feature_enabled("VERSIONED_EXPORT"):
timestamp = datetime.now().strftime("%Y%m%dT%H%M%S")
root = f"dashboard_export_{timestamp}"
filename = f"{root}.zip"
buf = BytesIO()
with ZipFile(buf, "w") as bundle:
try:
for file_name, file_content in ExportDashboardsCommand(
requested_ids
).run():
with bundle.open(f"{root}/{file_name}", "w") as fp:
fp.write(file_content.encode())
except DashboardNotFoundError:
return self.response_404()
buf.seek(0)
return send_file(
buf,
mimetype="application/zip",
as_attachment=True,
attachment_filename=filename,
)
query = self.datamodel.session.query(Dashboard).filter(
Dashboard.id.in_(requested_ids)
)
query = self._base_filters.apply_all(query)
ids = [item.id for item in query.all()]
if not ids:
return self.response_404()
export = Dashboard.export_dashboards(ids)
resp = make_response(export, 200)
resp.headers["Content-Disposition"] = generate_download_headers("json")[
"Content-Disposition"
]
return resp
def testppt():
out_file = BytesIO()
imagefile = BytesIO()
# imagefile = StringIO()
c = create_chart_object()
c.save(imagefile, 'png', scale_factor=1.9)
imagefile.seek(0)
path = 'template.pptx'
prs = Presentation(path)
# blank_slide_layout = prs.slide_layouts[6]
# slide = prs.slides.add_slide(blank_slide_layout)
title_only_slide_layout = prs.slide_layouts[4]
slide = prs.slides.add_slide(title_only_slide_layout)
shapes = slide.shapes
shapes.title.text = 'Adding an AutoShape'
left = Pt(50)
top = Pt(300)
width = Pt(700)
height = Pt(350)
txBox = shapes.add_textbox(left, top, width, height)
tf = txBox.text_frame
tf.auto_size = MSO_AUTO_SIZE.SHAPE_TO_FIT_TEXT
tf.word_wrap = True
tf.clear() # remove any existing paragraphs, leaving one empty one
p = tf.paragraphs[0]
p.alignment = PP_ALIGN.LEFT
p.text = "LS 1234567"
p.font.bold = True
p = tf.add_paragraph()
p.alignment = PP_ALIGN.LEFT
p.text = "This is a second paragraph containing some comment"
# top = Inches(1.5)
# left = Inches(5)
top = Pt(300)
left = Pt(900)
pic = shapes.add_picture(imagefile, left, top, height=Pt(500))
symptoms = [{
"abbr": "A",
"symp": "bubu"
}, {
"abbr": "B",
"symp": "bibi"
}, {
"abbr": "CA",
"symp": "lala"
}]
rows = len(symptoms) + 1
cols = 2
left = Pt(50)
top = Pt(650)
width = Pt(350)
height = Pt(150)
table = shapes.add_table(rows, cols, left, top, width, height).table
# set column widths
table.columns[0].width = Pt(70)
table.columns[1].width = Pt(300)
table.cell(0, 0).text = 'Abbreviation'
table.cell(0, 1).text = 'Symptom'
for i in range(rows - 1):
table.cell(i, 0).text = symptoms[i]['abbr']
table.cell(i, 1).text = symptoms[i]['symp']
prs.save(out_file)
out_file.seek(0)
return send_file(out_file,
attachment_filename="testing.pptx",
as_attachment=True)
def test_complete():
fig = plt.figure('Figure with a label?', figsize=(10, 6))
plt.suptitle('Can you fit any more in a figure?')
# make some arbitrary data
x, y = np.arange(8), np.arange(10)
data = u = v = np.linspace(0, 10, 80).reshape(10, 8)
v = np.sin(v * -0.6)
# Ensure lists also pickle correctly.
plt.subplot(3, 3, 1)
plt.plot(list(range(10)))
plt.subplot(3, 3, 2)
plt.contourf(data, hatches=['//', 'ooo'])
plt.colorbar()
plt.subplot(3, 3, 3)
plt.pcolormesh(data)
plt.subplot(3, 3, 4)
plt.imshow(data)
plt.subplot(3, 3, 5)
plt.pcolor(data)
ax = plt.subplot(3, 3, 6)
ax.set_xlim(0, 7)
ax.set_ylim(0, 9)
plt.streamplot(x, y, u, v)
ax = plt.subplot(3, 3, 7)
ax.set_xlim(0, 7)
ax.set_ylim(0, 9)
plt.quiver(x, y, u, v)
plt.subplot(3, 3, 8)
plt.scatter(x, x**2, label='$x^2$')
plt.legend(loc='upper left')
plt.subplot(3, 3, 9)
plt.errorbar(x, x * -0.5, xerr=0.2, yerr=0.4)
#
# plotting is done, now test its pickle-ability
#
result_fh = BytesIO()
pickle.dump(fig, result_fh, pickle.HIGHEST_PROTOCOL)
plt.close('all')
# make doubly sure that there are no figures left
assert plt._pylab_helpers.Gcf.figs == {}
# wind back the fh and load in the figure
result_fh.seek(0)
fig = pickle.load(result_fh)
# make sure there is now a figure manager
assert plt._pylab_helpers.Gcf.figs != {}
assert fig.get_label() == 'Figure with a label?'
def prediction_interval_plot_as_png_base64(df,
time,
signal,
estimator,
lower,
upper,
name=None,
max_length=1000):
assert df.shape[0] > 0
assert df.shape[1] > 0
assert time in df.columns
assert signal in df.columns
assert estimator in df.columns
assert lower in df.columns
assert upper in df.columns
df1 = df.tail(max_length).copy()
lMin = np.mean(df1[signal]) - np.std(df1[signal]) * 3
lMax = np.mean(df1[signal]) + np.std(df1[signal]) * 3
df1[lower] = df1[lower].apply(lambda x: x
if (np.isnan(x) or x >= lMin) else np.nan)
df1[upper] = df1[upper].apply(lambda x: x
if (np.isnan(x) or x <= lMax) else np.nan)
# last value of the signal
lLastSignalPos = df1[signal].dropna().tail(1).index
lEstimtorValue = df1[estimator][lLastSignalPos]
df1.loc[lLastSignalPos, lower] = lEstimtorValue
df1.loc[lLastSignalPos, upper] = lEstimtorValue
import matplotlib
# matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.switch_backend("Agg")
fig, axs = plt.subplots(ncols=1, figsize=(16, 8))
df1.plot.line(
time,
[signal, estimator, lower, upper],
color=[SIGNAL_COLOR, FORECAST_COLOR, FORECAST_COLOR, FORECAST_COLOR],
ax=axs,
grid=True,
legend=False,
)
add_patched_legend(axs, [signal, estimator, lower, upper])
x = df1[time]
type1 = np.dtype(x)
if type1.kind == "M":
x = x.apply(lambda t: t.date())
axs.fill_between(x.values,
df1[lower],
df1[upper],
color=SHADED_COLOR,
alpha=0.5)
figfile = BytesIO()
fig.savefig(figfile, format="png")
plt.close(fig)
figfile.seek(0) # rewind to beginning of file
figdata_png = base64.b64encode(figfile.getvalue())
return figdata_png.decode("utf8")
def wav_info_to_csv():
'''
To save wav files details to csv of a directory
'''
wav_info_tags = ["Filename", "Operator", "DeviceID", "Battery_Voltage", "Battery_Percentage",
"Network_status", "Firmare_Revision", "Time_Stamp", "Latitude", "Longitude",
"ChunkID", "TotalSize", "Format", "SubChunk1ID", "SubChunk1Size",
"AudioFormat", "NumChannels", "SampleRate", "ByteRate", "BlockAlign",
"BitsPerSample", "SubChunk2ID", "SubChunk2Size"]
initial_number_of_devices = 0
count = 0
DICT = {}
wav_file_name = "wav_file_extra_information_"+CURRENT_PATH.split("/")[-1]+".csv"
print wav_file_name
with open(wav_file_name, "w") as file_object:
wav_information_object = csv.writer(file_object)
wav_information_object.writerow(wav_info_tags)
file_object.flush()
for name in NAMES:
if (name[-3:] == 'wav') or (name[-3:] == 'WAV'):
time1 = ftp.voidcmd("MDTM " + name)
count += 1
DICT[name] = time1[4:]
sorted_wav_files_list = sorted((value,key) for (key,value) in DICT.items())
sorted_file_name_list = [element[1] for element in sorted_wav_files_list]
for name in sorted_file_name_list:
if (name[-3:] == 'wav') or (name[-3:] == 'WAV'):
file_header_info = BytesIO(FtpFile(ftp, name).read(264))
riff, size, fformat = struct.unpack('<4sI4s', file_header_info.read(12))
chunkoffset = file_header_info.tell()
chunk_header = file_header_info.read(8)
subchunkid, subchunksize = struct.unpack('<4sI', chunk_header)
chunkoffset = file_header_info.tell()
aformat, channels, samplerate, byterate, blockalign, bps = struct.unpack('HHIIHH', file_header_info.read(16))
chunkoffset = file_header_info.tell()
struct.unpack('<4sI', file_header_info.read(8))
struct.unpack('<4s4sI', file_header_info.read(12))
chunkoffset = file_header_info.tell()
extra_header = file_header_info.read(200)
chunkoffset = file_header_info.tell()
file_header_info.seek(chunkoffset)
subchunk2id, subchunk2size = struct.unpack('<4sI', file_header_info.read(8))
chunkoffset = file_header_info.tell()
wav_header = riff, size, fformat, subchunkid, subchunksize, aformat, \
channels, samplerate, byterate, blockalign, bps, subchunk2id, subchunk2size
#Getting the wav information and writing the csv file rows
wav_information = extra_header.decode("ascii").split(',')
information_value = [name]
for index_value, each_tag_value in enumerate(wav_information):
try:
_, corresponding_value = each_tag_value.split(":")
except ValueError:
corresponding_value = "".join(each_tag_value.split(":")[1:])
information_value.append(corresponding_value)
for info in wav_header:
information_value.append(info)
wav_information_object.writerow(information_value)
file_object.flush()
print "wav file information saved to csv.."
def receive(self, command=None, prompts=None, answer=None, newline=True, prompt_retry_check=False, check_all=False):
'''
Handles receiving of output from command
'''
self._matched_prompt = None
self._matched_cmd_prompt = None
recv = BytesIO()
handled = False
command_prompt_matched = False
matched_prompt_window = window_count = 0
command_timeout = self.get_option('persistent_command_timeout')
self._validate_timeout_value(command_timeout, "persistent_command_timeout")
buffer_read_timeout = self.get_option('persistent_buffer_read_timeout')
self._validate_timeout_value(buffer_read_timeout, "persistent_buffer_read_timeout")
while True:
if command_prompt_matched:
try:
signal.signal(signal.SIGALRM, self._handle_buffer_read_timeout)
signal.setitimer(signal.ITIMER_REAL, buffer_read_timeout)
data = self._ssh_shell.recv(256)
signal.alarm(0)
# if data is still received on channel it indicates the prompt string
# is wrongly matched in between response chunks, continue to read
# remaining response.
command_prompt_matched = False
# restart command_timeout timer
signal.signal(signal.SIGALRM, self._handle_command_timeout)
signal.alarm(command_timeout)
except AnsibleCmdRespRecv:
return self._command_response
else:
data = self._ssh_shell.recv(256)
# when a channel stream is closed, received data will be empty
if not data:
break
recv.write(data)
offset = recv.tell() - 256 if recv.tell() > 256 else 0
recv.seek(offset)
window = self._strip(recv.read())
window_count += 1
if prompts and not handled:
handled = self._handle_prompt(window, prompts, answer, newline, False, check_all)
matched_prompt_window = window_count
elif prompts and handled and prompt_retry_check and matched_prompt_window + 1 == window_count:
# check again even when handled, if same prompt repeats in next window
# (like in the case of a wrong enable password, etc) indicates
# value of answer is wrong, report this as error.
if self._handle_prompt(window, prompts, answer, newline, prompt_retry_check, check_all):
raise AnsibleConnectionFailure("For matched prompt '%s', answer is not valid" % self._matched_cmd_prompt)
if self._find_prompt(window):
self._last_response = recv.getvalue()
resp = self._strip(self._last_response)
self._command_response = self._sanitize(resp, command)
if buffer_read_timeout == 0.0:
return self._command_response
else:
command_prompt_matched = True
def testplot(names, showChannels, infile, outfile, approved):
x = []
ch1 = []
ch2 = []
ch3 = []
ch4 = []
ch5 = []
ch6 = []
ch7 = []
ch8 = []
with open(infile, 'r') as csvfile:
plots = csv.reader(csvfile, delimiter=',')
fig, ax = plt.subplots()
df = pd.read_csv(infile,
sep='\s*,\s*',
encoding='ascii',
engine='python')
#df.set_index("Time", inplace=True)
#print(df.head())
headers = df.columns.tolist()
#print(headers)
#for row in plots:
# ch1.append(int(row[1]))
test = df[headers[0]]
print(df[headers[0]].head())
if showChannels[0] == 'true':
plt.plot(df[headers[0]], df[headers[1]], label=headers[1])
if showChannels[1] == 'true':
plt.plot(df[headers[0]], df[headers[2]], label=headers[2])
if showChannels[2] == 'true':
plt.plot(df[headers[0]], df[headers[3]], label=headers[3])
if showChannels[3] == 'true':
plt.plot(df[headers[0]], df[headers[4]], label=headers[4])
if showChannels[4] == 'true':
plt.plot(df[headers[0]], df[headers[5]], label=headers[5])
if showChannels[5] == 'true':
plt.plot(df[headers[0]], df[headers[6]], label=headers[6])
if showChannels[6] == 'true':
plt.plot(df[headers[0]], df[headers[7]], label=headers[7])
if showChannels[7] == 'true':
plt.plot(df[headers[0]], df[headers[8]], label=headers[8])
ax.xaxis.set_major_locator(plt.MaxNLocator(3))
ax.set_ylim(0, 1000)
plt.setp(ax.get_xticklabels())
plt.xlabel('time')
plt.grid()
plt.ylabel('mbar')
plt.legend()
imgdata = BytesIO()
plt.savefig(imgdata, format='png', dpi=1200)
imgdata.seek(0)
im = ImageReader(imgdata)
logo = ImageReader('/home/larskf/TestReportGenerator/DB_long.jpg')
c = canvas.Canvas(outfile)
w, h = c._pagesize
c.drawImage(im, 0, 120, w, h, preserveAspectRatio=True)
c.drawImage(logo, 100, h / 2 - 50, w - 200, h, preserveAspectRatio=True)
start = datetime.datetime.strptime(df[headers[0]].values[0],
'%d.%m.%Y %H:%M:%S')
stop = datetime.datetime.strptime(df[headers[0]].values[-1],
'%d.%m.%Y %H:%M:%S')
dur = stop - start
c.setFont('Helvetica-Bold', 13)
str = "Pressure leak test"
c.drawString(50, 725, str)
c.setFont('Helvetica', 11)
str = "Started logging at:"
c.drawString(50, 300, str)
str = " {}".format(start)
c.drawString(50, 280, str)
str = "Test duration: {}".format(dur)
c.drawString(50, 260, str)
c.drawString(
50, 255,
"_______________________________________________________________________________"
)
c.setFont('Helvetica', 9)
if showChannels[0] == 'true':
maxval = max(df[headers[1]])
minval = min(df[headers[1]])
avgval = statistics.mean(df[headers[1]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[1], maxval, minval, avgval, maxval - minval)
#str = "%s: Max: %f.2 Min: %f.2 Avg: %f.2 " % (names[0], maxval, minval, avgval)
c.drawString(50, 240, str)
if showChannels[1] == 'true':
maxval = max(df[headers[2]])
minval = min(df[headers[2]])
avgval = statistics.mean(df[headers[2]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[2], maxval, minval, avgval, maxval - minval)
c.drawString(50, 220, str)
if showChannels[2] == 'true':
maxval = max(df[headers[3]])
minval = min(df[headers[3]])
avgval = statistics.mean(df[headers[3]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[3], maxval, minval, avgval, maxval - minval)
c.drawString(50, 200, str)
if showChannels[3] == 'true':
maxval = max(df[headers[4]])
minval = min(df[headers[4]])
avgval = statistics.mean(df[headers[4]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[4], maxval, minval, avgval, maxval - minval)
c.drawString(50, 180, str)
if showChannels[4] == 'true':
maxval = max(df[headers[5]])
minval = min(df[headers[5]])
avgval = statistics.mean(df[headers[5]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[5], maxval, minval, avgval, maxval - minval)
c.drawString(50, 160, str)
if showChannels[5] == 'true':
maxval = max(df[headers[6]])
minval = min(df[headers[6]])
avgval = statistics.mean(df[headers[6]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[6], maxval, minval, avgval, maxval - minval)
c.drawString(50, 140, str)
if showChannels[6] == 'true':
maxval = max(df[headers[7]])
minval = min(df[headers[7]])
avgval = statistics.mean(df[headers[7]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[7], maxval, minval, avgval, maxval - minval)
c.drawString(50, 120, str)
if showChannels[7] == 'true':
maxval = max(df[headers[8]])
minval = min(df[headers[8]])
avgval = statistics.mean(df[headers[8]])
str = "{}: Max: {:.0f} mbar Min: {:.0f} mbar Avg: {:.0f} mbar Drop: {:.0f} mbar".format(
headers[8], maxval, minval, avgval, maxval - minval)
c.drawString(50, 100, str)
c.drawString(
50, 95,
"_______________________________________________________________________________"
)
c.setFont('Helvetica', 11)
str = "Approved by: %s" % approved
c.drawString(50, 75, str)
c.drawString(50, 30, "_____________________________________")
c.save()
def plot_ROC(X, y, classifier, cv):
from sklearn.metrics import roc_curve, auc
from sklearn.model_selection import StratifiedKFold
from scipy import interp
cv = StratifiedKFold(n_splits=cv)
tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)
i = 0
for train, test in cv.split(X, y):
probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
# Compute ROC curve and area the curve
fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
tprs.append(interp(mean_fpr, fpr, tpr))
tprs[-1][0] = 0.0
roc_auc = auc(fpr, tpr)
aucs.append(roc_auc)
plt.plot(fpr,
tpr,
lw=1,
alpha=0.3,
label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))
i += 1
#figure = plt.figure()
plt.gcf().clear()
plt.plot([0, 1], [0, 1],
linestyle='--',
lw=2,
color='r',
label='Luck',
alpha=.8)
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
plt.plot(mean_fpr,
mean_tpr,
color='b',
label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
lw=2,
alpha=.8)
std_tpr = np.std(tprs, axis=0)
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
plt.fill_between(mean_fpr,
tprs_lower,
tprs_upper,
color='grey',
alpha=.2,
label=r'$\pm$ 1 std. dev.')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
from io import BytesIO
figfile = BytesIO()
plt.savefig(figfile, format='png')
figfile.seek(0) # rewind to beginning of file
import base64
figdata_png = base64.b64encode(figfile.getvalue())
return figdata_png
def iterate_over_test_chunk(pyarrow_type,
column_meta,
source_data_generator,
expected_data_transformer=None):
stream = BytesIO()
assert len(pyarrow_type) == len(column_meta)
column_size = len(pyarrow_type)
batch_row_count = 10
batch_count = 9
fields = []
for i in range(column_size):
fields.append(
pyarrow.field(f"column_{i}", pyarrow_type[i], True,
column_meta[i]))
schema = pyarrow.schema(fields)
expected_data = []
writer = RecordBatchStreamWriter(stream, schema)
for i in range(batch_count):
column_arrays = []
py_arrays = []
for j in range(column_size):
column_data = []
not_none_cnt = 0
while not_none_cnt == 0:
column_data = []
for _ in range(batch_row_count):
data = (None if bool(random.getrandbits(1)) else
source_data_generator())
if data is not None:
not_none_cnt += 1
column_data.append(data)
column_arrays.append(column_data)
py_arrays.append(pyarrow.array(column_data, type=pyarrow_type[j]))
if expected_data_transformer:
for i in range(len(column_arrays)):
column_arrays[i] = [
expected_data_transformer(_data)
if _data is not None else None
for _data in column_arrays[i]
]
expected_data.append(column_arrays)
column_names = [f"column_{i}" for i in range(column_size)]
rb = RecordBatch.from_arrays(py_arrays, column_names)
writer.write_batch(rb)
writer.close()
# seek stream to begnning so that we can read from stream
stream.seek(0)
context = ArrowConverterContext()
it = PyArrowIterator(None, stream, context, False, False, False)
it.init(IterUnit.ROW_UNIT.value)
count = 0
while True:
try:
val = next(it)
for i in range(column_size):
batch_index = int(count / batch_row_count)
assert (val[i] == expected_data[batch_index][i][
count - batch_row_count * batch_index])
count += 1
except StopIteration:
assert count == (batch_count * batch_row_count)
break
def _rt_check_case(name, expected, format):
mat_stream = BytesIO()
savemat(mat_stream, expected, format=format)
mat_stream.seek(0)
_load_check_case(name, [mat_stream], expected)
class THeaderTransport(TTransportBase, CReadableTransport):
"""Transport that sends headers. Also understands framed/unframed/HTTP
transports and will do the right thing"""
__max_frame_size = MAX_FRAME_SIZE
# Defaults to current user, but there is also a setter below.
__identity = None
__first_request = True
IDENTITY_HEADER = "identity"
ID_VERSION_HEADER = "id_version"
ID_VERSION = "1"
CLIENT_METADATA_HEADER = "client_metadata"
def __init__(self, trans, client_types=None, client_type=None):
self.__trans = trans
self.__rbuf = StringIO()
self.__rbuf_frame = False
self.__wbuf = StringIO()
self.seq_id = 0
self.__flags = 0
self.__read_transforms = []
self.__write_transforms = []
self.__supported_client_types = set(client_types or (CLIENT_TYPE.HEADER,))
self.__proto_id = T_COMPACT_PROTOCOL # default to compact like c++
self.__client_type = client_type or CLIENT_TYPE.HEADER
self.__read_headers = {}
self.__read_persistent_headers = {}
self.__write_headers = {}
self.__write_persistent_headers = {}
if self.__client_type in (
CLIENT_TYPE.UNFRAMED_DEPRECATED,
CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED,
CLIENT_TYPE.FRAMED_DEPRECATED,
CLIENT_TYPE.FRAMED_COMPACT,
):
self.__client_type = CLIENT_TYPE.HEADER
self.__supported_client_types.add(self.__client_type)
# If we support unframed binary / framed binary also support compact
if CLIENT_TYPE.UNFRAMED_DEPRECATED in self.__supported_client_types:
self.__supported_client_types.add(CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED)
if CLIENT_TYPE.FRAMED_DEPRECATED in self.__supported_client_types:
self.__supported_client_types.add(CLIENT_TYPE.FRAMED_COMPACT)
def set_header_flag(self, flag):
self.__flags |= flag
def clear_header_flag(self, flag):
self.__flags &= ~flag
def header_flags(self):
return self.__flags
def set_max_frame_size(self, size):
if size > MAX_BIG_FRAME_SIZE:
raise TTransportException(
TTransportException.INVALID_FRAME_SIZE,
"Cannot set max frame size > %s" % MAX_BIG_FRAME_SIZE,
)
if size > MAX_FRAME_SIZE and self.__client_type != CLIENT_TYPE.HEADER:
raise TTransportException(
TTransportException.INVALID_FRAME_SIZE,
"Cannot set max frame size > %s for clients other than HEADER"
% MAX_FRAME_SIZE,
)
self.__max_frame_size = size
def get_peer_identity(self):
if self.IDENTITY_HEADER in self.__read_headers:
if self.__read_headers[self.ID_VERSION_HEADER] == self.ID_VERSION:
return self.__read_headers[self.IDENTITY_HEADER]
return None
def set_identity(self, identity):
self.__identity = identity
def get_protocol_id(self):
return self.__proto_id
def set_protocol_id(self, proto_id):
self.__proto_id = proto_id
def set_header(self, str_key, str_value):
self.__write_headers[str_key] = str_value
def get_write_headers(self):
return self.__write_headers
def get_headers(self):
return self.__read_headers
def clear_headers(self):
self.__write_headers.clear()
def set_persistent_header(self, str_key, str_value):
self.__write_persistent_headers[str_key] = str_value
def get_write_persistent_headers(self):
return self.__write_persistent_headers
def clear_persistent_headers(self):
self.__write_persistent_headers.clear()
def add_transform(self, trans_id):
self.__write_transforms.append(trans_id)
def _reset_protocol(self):
# HTTP calls that are one way need to flush here.
if self.__client_type == CLIENT_TYPE.HTTP_SERVER:
self.flush()
# set to anything except unframed
self.__client_type = CLIENT_TYPE.UNKNOWN
# Read header bytes to check which protocol to decode
self.readFrame(0)
def getTransport(self):
return self.__trans
def isOpen(self):
return self.getTransport().isOpen()
def open(self):
return self.getTransport().open()
def close(self):
return self.getTransport().close()
def read(self, sz):
ret = self.__rbuf.read(sz)
if len(ret) == sz:
return ret
if self.__client_type in (
CLIENT_TYPE.UNFRAMED_DEPRECATED,
CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED,
):
return ret + self.getTransport().readAll(sz - len(ret))
self.readFrame(sz - len(ret))
return ret + self.__rbuf.read(sz - len(ret))
readAll = read # TTransportBase.readAll does a needless copy here.
def readFrame(self, req_sz):
self.__rbuf_frame = True
word1 = self.getTransport().readAll(4)
sz = unpack("!I", word1)[0]
proto_id = word1[0]
if proto_id == TBinaryProtocol.PROTOCOL_ID:
# unframed
self.__client_type = CLIENT_TYPE.UNFRAMED_DEPRECATED
self.__proto_id = T_BINARY_PROTOCOL
if req_sz <= 4: # check for reads < 0.
self.__rbuf = StringIO(word1)
else:
self.__rbuf = StringIO(word1 + self.getTransport().read(req_sz - 4))
elif proto_id == TCompactProtocol.PROTOCOL_ID:
self.__client_type = CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED
self.__proto_id = T_COMPACT_PROTOCOL
if req_sz <= 4: # check for reads < 0.
self.__rbuf = StringIO(word1)
else:
self.__rbuf = StringIO(word1 + self.getTransport().read(req_sz - 4))
elif sz == HTTP_SERVER_MAGIC:
self.__client_type = CLIENT_TYPE.HTTP_SERVER
mf = self.getTransport().handle.makefile("rb", -1)
self.handler = RequestHandler(mf, "client_address:port", "")
self.header = self.handler.wfile
self.__rbuf = StringIO(self.handler.data)
else:
if sz == BIG_FRAME_MAGIC:
sz = unpack("!Q", self.getTransport().readAll(8))[0]
# could be header format or framed. Check next two bytes.
magic = self.getTransport().readAll(2)
proto_id = magic[0]
if proto_id == TCompactProtocol.PROTOCOL_ID:
self.__client_type = CLIENT_TYPE.FRAMED_COMPACT
self.__proto_id = T_COMPACT_PROTOCOL
_frame_size_check(sz, self.__max_frame_size, header=False)
self.__rbuf = StringIO(magic + self.getTransport().readAll(sz - 2))
elif proto_id == TBinaryProtocol.PROTOCOL_ID:
self.__client_type = CLIENT_TYPE.FRAMED_DEPRECATED
self.__proto_id = T_BINARY_PROTOCOL
_frame_size_check(sz, self.__max_frame_size, header=False)
self.__rbuf = StringIO(magic + self.getTransport().readAll(sz - 2))
elif magic == PACKED_HEADER_MAGIC:
self.__client_type = CLIENT_TYPE.HEADER
_frame_size_check(sz, self.__max_frame_size)
# flags(2), seq_id(4), header_size(2)
n_header_meta = self.getTransport().readAll(8)
self.__flags, self.seq_id, header_size = unpack("!HIH", n_header_meta)
data = StringIO()
data.write(magic)
data.write(n_header_meta)
data.write(self.getTransport().readAll(sz - 10))
data.seek(10)
self.read_header_format(sz - 10, header_size, data)
else:
self.__client_type = CLIENT_TYPE.UNKNOWN
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE,
"Could not detect client transport type",
)
if self.__client_type not in self.__supported_client_types:
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE,
"Client type {} not supported on server".format(self.__client_type),
)
def read_header_format(self, sz, header_size, data):
# clear out any previous transforms
self.__read_transforms = []
header_size = header_size * 4
if header_size > sz:
raise TTransportException(
TTransportException.INVALID_FRAME_SIZE,
"Header size is larger than frame",
)
end_header = header_size + data.tell()
self.__proto_id = readVarint(data)
num_headers = readVarint(data)
if self.__proto_id == 1 and self.__client_type != CLIENT_TYPE.HTTP_SERVER:
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE,
"Trying to recv JSON encoding over binary",
)
# Read the headers. Data for each header varies.
for _ in range(0, num_headers):
trans_id = readVarint(data)
if trans_id in (TRANSFORM.ZLIB, TRANSFORM.SNAPPY, TRANSFORM.ZSTD):
self.__read_transforms.insert(0, trans_id)
elif trans_id == TRANSFORM.HMAC:
raise TApplicationException(
TApplicationException.INVALID_TRANSFORM,
"Hmac transform is no longer supported: %i" % trans_id,
)
else:
# TApplicationException will be sent back to client
raise TApplicationException(
TApplicationException.INVALID_TRANSFORM,
"Unknown transform in client request: %i" % trans_id,
)
# Clear out previous info headers.
self.__read_headers.clear()
# Read the info headers.
while data.tell() < end_header:
info_id = readVarint(data)
if info_id == INFO.NORMAL:
_read_info_headers(data, end_header, self.__read_headers)
elif info_id == INFO.PERSISTENT:
_read_info_headers(data, end_header, self.__read_persistent_headers)
else:
break # Unknown header. Stop info processing.
if self.__read_persistent_headers:
self.__read_headers.update(self.__read_persistent_headers)
# Skip the rest of the header
data.seek(end_header)
payload = data.read(sz - header_size)
# Read the data section.
self.__rbuf = StringIO(self.untransform(payload))
def write(self, buf):
self.__wbuf.write(buf)
def transform(self, buf):
for trans_id in self.__write_transforms:
if trans_id == TRANSFORM.ZLIB:
buf = zlib.compress(buf)
elif trans_id == TRANSFORM.SNAPPY:
buf = snappy.compress(buf)
elif trans_id == TRANSFORM.ZSTD:
buf = zstd.ZstdCompressor(write_content_size=True).compress(buf)
else:
raise TTransportException(
TTransportException.INVALID_TRANSFORM,
"Unknown transform during send",
)
return buf
def untransform(self, buf):
for trans_id in self.__read_transforms:
if trans_id == TRANSFORM.ZLIB:
buf = zlib.decompress(buf)
elif trans_id == TRANSFORM.SNAPPY:
buf = snappy.decompress(buf)
elif trans_id == TRANSFORM.ZSTD:
buf = zstd.ZstdDecompressor().decompress(buf)
if trans_id not in self.__write_transforms:
self.__write_transforms.append(trans_id)
return buf
def disable_client_metadata(self):
self.__first_request = False
def flush(self):
self.flushImpl(False)
def onewayFlush(self):
self.flushImpl(True)
def _flushHeaderMessage(self, buf, wout, wsz):
"""Write a message for CLIENT_TYPE.HEADER
@param buf(StringIO): Buffer to write message to
@param wout(str): Payload
@param wsz(int): Payload length
"""
transform_data = StringIO()
# For now, all transforms don't require data.
num_transforms = len(self.__write_transforms)
for trans_id in self.__write_transforms:
transform_data.write(getVarint(trans_id))
# Add in special flags.
if self.__identity:
self.__write_headers[self.ID_VERSION_HEADER] = self.ID_VERSION
self.__write_headers[self.IDENTITY_HEADER] = self.__identity
if self.__first_request:
self.__first_request = False
self.__write_headers[
self.CLIENT_METADATA_HEADER
] = '{"agent":"THeaderTransport.py"}'
info_data = StringIO()
# Write persistent kv-headers
_flush_info_headers(
info_data, self.get_write_persistent_headers(), INFO.PERSISTENT
)
# Write non-persistent kv-headers
_flush_info_headers(info_data, self.__write_headers, INFO.NORMAL)
header_data = StringIO()
header_data.write(getVarint(self.__proto_id))
header_data.write(getVarint(num_transforms))
header_size = transform_data.tell() + header_data.tell() + info_data.tell()
padding_size = 4 - (header_size % 4)
header_size = header_size + padding_size
# MAGIC(2) | FLAGS(2) + SEQ_ID(4) + HEADER_SIZE(2)
wsz += header_size + 10
if wsz > MAX_FRAME_SIZE:
buf.write(pack("!I", BIG_FRAME_MAGIC))
buf.write(pack("!Q", wsz))
else:
buf.write(pack("!I", wsz))
buf.write(pack("!HH", HEADER_MAGIC >> 16, self.__flags))
buf.write(pack("!I", self.seq_id))
buf.write(pack("!H", header_size // 4))
buf.write(header_data.getvalue())
buf.write(transform_data.getvalue())
buf.write(info_data.getvalue())
# Pad out the header with 0x00
for _ in range(0, padding_size, 1):
buf.write(pack("!c", b"\0"))
# Send data section
buf.write(wout)
def flushImpl(self, oneway):
wout = self.__wbuf.getvalue()
wout = self.transform(wout)
wsz = len(wout)
# reset wbuf before write/flush to preserve state on underlying failure
self.__wbuf.seek(0)
self.__wbuf.truncate()
if self.__proto_id == 1 and self.__client_type != CLIENT_TYPE.HTTP_SERVER:
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE,
"Trying to send JSON encoding over binary",
)
buf = StringIO()
if self.__client_type == CLIENT_TYPE.HEADER:
self._flushHeaderMessage(buf, wout, wsz)
elif self.__client_type in (
CLIENT_TYPE.FRAMED_DEPRECATED,
CLIENT_TYPE.FRAMED_COMPACT,
):
buf.write(pack("!i", wsz))
buf.write(wout)
elif self.__client_type in (
CLIENT_TYPE.UNFRAMED_DEPRECATED,
CLIENT_TYPE.UNFRAMED_COMPACT_DEPRECATED,
):
buf.write(wout)
elif self.__client_type == CLIENT_TYPE.HTTP_SERVER:
# Reset the client type if we sent something -
# oneway calls via HTTP expect a status response otherwise
buf.write(self.header.getvalue())
buf.write(wout)
self.__client_type == CLIENT_TYPE.HEADER
elif self.__client_type == CLIENT_TYPE.UNKNOWN:
raise TTransportException(
TTransportException.INVALID_CLIENT_TYPE, "Unknown client type"
)
# We don't include the framing bytes as part of the frame size check
frame_size = buf.tell() - (4 if wsz < MAX_FRAME_SIZE else 12)
_frame_size_check(
frame_size,
self.__max_frame_size,
header=self.__client_type == CLIENT_TYPE.HEADER,
)
self.getTransport().write(buf.getvalue())
if oneway:
self.getTransport().onewayFlush()
else:
self.getTransport().flush()
# Implement the CReadableTransport interface.
@property
def cstringio_buf(self):
if not self.__rbuf_frame:
self.readFrame(0)
return self.__rbuf
def cstringio_refill(self, prefix, reqlen):
# self.__rbuf will already be empty here because fastproto doesn't
# ask for a refill until the previous buffer is empty. Therefore,
# we can start reading new frames immediately.
# On unframed clients, there is a chance there is something left
# in rbuf, and the read pointer is not advanced by fastproto
# so seek to the end to be safe
self.__rbuf.seek(0, 2)
while len(prefix) < reqlen:
prefix += self.read(reqlen)
self.__rbuf = StringIO(prefix)
return self.__rbuf
async def draw_matches_table(matches, game_strings):
region_data = read_json(settings.resource("json/region_data.json"))
border_size = 10
grey_color = "#BBBBBB"
table = Table(background=discord_color2)
# Header
headers = [
TextCell("Hero", padding=0),
TextCell(""),
TextCell("Result"),
TextCell("K", horizontal_align="center"),
TextCell("D", horizontal_align="center"),
TextCell("A", horizontal_align="center"),
TextCell("Duration"),
TextCell("Type"),
TextCell("Date")
]
table.add_row(headers)
for cell in table.rows[0]:
cell.background = discord_color1
table.add_row([
ColorCell(color=discord_color1, height=6) for i in range(len(headers))
])
first = True
for match in matches:
won_match = bool(
match["radiant_win"]) == bool(match["player_slot"] < 128)
game_mode = game_strings.get(f"game_mode_{match['game_mode']}",
"Unknown")
lobby_type = game_strings.get(f"lobby_type_{match['lobby_type']}",
"Unknown")
if first:
first = False
else:
table.add_row([
ColorCell(color=discord_color2, height=12)
for i in range(len(headers))
])
table.add_row([
ImageCell(img=await get_hero_image(match["hero_id"]), height=48),
DoubleCell(
TextCell(get_hero_name(match["hero_id"]), font_size=24),
TextCell(match.get("match_id"),
font_size=12,
horizontal_align="left",
color=grey_color)),
TextCell("Win" if won_match else "Loss",
color=("green" if won_match else "red"),
horizontal_align="center"),
TextCell(match.get("kills")),
TextCell(match.get("deaths")),
TextCell(match.get("assists")),
TextCell(format_duration_simple(match.get("duration")),
horizontal_align="center"),
DoubleCell(
TextCell(game_mode,
font_size=18,
padding_right=15,
color=grey_color),
TextCell(lobby_type,
font_size=18,
padding_right=15,
color=grey_color)),
get_datetime_cell(match, region_data)
])
image = table.render()
border_image = Image.new('RGBA',
(image.size[0] +
(border_size * 2), image.size[1] + border_size),
color=discord_color1)
image = paste_image(border_image, image, border_size, 0)
fp = BytesIO()
image.save(fp, format="PNG")
fp.seek(0)
return fp
def detect_sync(*args, **kwargs):
# check if the request_processing_function had an error while parsing user specified parameters
if kwargs.get('error_code', None) is not None:
return _make_error_response(kwargs.get('error_code'),
kwargs.get('error_message'))
render_boxes = kwargs.get('render_boxes')
classification = kwargs.get('classification')
detection_confidence = kwargs.get('detection_confidence')
images = kwargs.get('images')
image_names = kwargs.get('image_names')
detection_results = []
inference_time_detector = []
try:
print('runserver, post_detect_sync, scoring images...')
for image_name, image in zip(image_names, images):
start_time = time.time()
result = detector.generate_detections_one_image(image, image_name)
detection_results.append(result)
elapsed = time.time() - start_time
inference_time_detector.append(elapsed)
except Exception as e:
print('Error performing detection on the images: ' + str(e))
log.log_exception('Error performing detection on the images: ' +
str(e))
return _make_error_response(
500, 'Error performing detection on the images: ' + str(e))
# filter the detections by the confidence threshold
filtered_results = {
} # json to return to the user along with the rendered images if they opted for it
# each result is [ymin, xmin, ymax, xmax, confidence, category]
try:
for result in detection_results:
image_name = result['file']
detections = result.get('detections', None)
filtered_results[image_name] = []
if detections is None:
continue
for d in detections:
if d['conf'] > detection_confidence:
res = TFDetector.convert_to_tf_coords(d['bbox'])
res.append(d['conf'])
res.append(
int(d['category'])
) # category is an int here, not string as in the async API
filtered_results[image_name].append(res)
except Exception as e:
print('Error consolidating the detection boxes: ' + str(e))
log.log_exception('Error consolidating the detection boxes: ' + str(e))
return _make_error_response(
500, 'Error consolidating the detection boxes: ' + str(e))
# classification
classification_result = {}
# TODO
# try:
# if classification:
# print('runserver, classification...')
# tic = datetime.now()
# classification_result = classifier.classify_boxes(images, image_names, result, classification)
# toc = datetime.now()
# classification_inference_duration = toc - tic
# print('runserver, classification, classifcation inference duraction: {}' \
# .format({classification_inference_duration}))
#
# else:
# classification_result = {}
#
# except Exception as e:
# print('Error performing classification on the images: ' + str(e))
# log.log_exception('Error performing classification on the images: ' + str(e))
# abort(500, 'Error performing classification on the images: ' + str(e))
# return results; optionally render the detections on the images and send the annotated images back
try:
print(
'runserver, post_detect_sync, rendering and sending images back...'
)
fields = {
'detection_result':
('detection_result', json.dumps(filtered_results),
'application/json'),
'classification_result':
('classification_result', json.dumps(classification_result),
'application/json')
}
if render_boxes:
for image_name, image, result in zip(image_names, images,
detection_results):
detections = result.get('detections', None)
if detections is None:
continue
viz_utils.render_detection_bounding_boxes(
detections,
image,
confidence_threshold=detection_confidence)
output_img_stream = BytesIO()
image.save(output_img_stream, format='jpeg')
output_img_stream.seek(0)
fields[image_name] = (image_name, output_img_stream,
'image/jpeg')
m = MultipartEncoder(fields=fields)
if len(inference_time_detector) > 0:
mean_inference_time_detector = sum(inference_time_detector) / len(
inference_time_detector)
else:
mean_inference_time_detector = -1
log.log_info(
'detector mean inference time',
mean_inference_time_detector,
additionalProperties={
'detector mean inference time':
str(mean_inference_time_detector),
# TODO 'classification mean inference time': str(''),
'num_images': len(image_names),
'render_boxes': render_boxes,
'detection_confidence': detection_confidence
})
return Response(m.to_string(), mimetype=m.content_type)
except Exception as e:
print('Error returning result or rendering the detection boxes: ' +
str(e))
log.log_exception(
'Error returning result or rendering the detection boxes: ' +
str(e))
return _make_error_response(
500, 'Error returning result or rendering the detection boxes: ' +
str(e))
def do_test_fn_return_output_socket(function, *args):
output_socket = BytesIO()
function(output_socket, *args)
output_socket.seek(0)
return output_socket
def rebuild(ds):
# pick and unpickle
f = BytesIO()
picked = pickle.dump(ds, f)
f.seek(0)
return pickle.load(f)
def img2bytes(img: Image.Image):
buf = BytesIO()
img.save(buf, format='PNG')
buf.seek(0)
return buf
