def handle_write(self): ''' @see: dispatcher.handle_write ''' assert self._writeq, 'Nothing to write' data, close = BytesIO(), False while self._writeq and data.tell() < self.bufferSize: content = self._writeq.popleft() if content is None: if self.close_connection: close = True break if isinstance(content, (bytes, memoryview)): data.write(content) elif isinstance(content, IInputStream): assert isinstance(content, IInputStream) byts = content.read(self.bufferSize - data.tell()) if byts == b'': if isinstance(content, IClosable): content.close() continue data.write(byts) self._writeq.appendleft(content) else: while data.tell() < self.bufferSize: try: byts = next(content) except StopIteration: break data.write(byts) sent = self.send(data.getbuffer()) if close: self.close() elif sent < data.tell(): self._writeq.appendleft(data.getbuffer()[sent:])
def _simulate_save(self, quality): """ Saves original_image to stream IO at given quality and returns the data and size """ output = BytesIO() save_kwargs = { 'format': self.original_format, self.quality_range[self.original_format]['meta']: quality } self.original_image.save( output, **save_kwargs ) print('Simulated save level %s size %s bytes' % (quality, output.getbuffer().nbytes)) return output, output.getbuffer().nbytes
def saveData(_id, zoom=15): """ read from CSV and parse data we use the database to store the data-key before it is will be intensely read """ csv = open("./%s/data.csv"%_id, "r") # data = mg.parseData(csv) data, info, header = mg.parseData(csv) csv.close() batch = leveldb.WriteBatch() for z in range(zoom+1): # splited_data = mg.splitData(data, z) splitted_data, splitted_info = mg.splitData(data, info, z) size = 2**z for x in range(size): for y in range(size): datum = np.array(splitted_data[x][y]) # key = dataKey(_id, x, y, z) datakey = dataKey(_id, x, y, z) value = BytesIO() np.save(value, datum) # batch.Put(key, value.getbuffer()) batch.Put(datakey, value.getbuffer()) infokey= infoKey(_id, x, y, z) batch.Put(infokey, json.dumps(splitted_info[x][y]).encode()) meta = {'header': header, 'color': '', 'infos': ''} writeMeta(_id, meta) db.Write(batch, sync=True)
def response_for_download_zip(folder, maxFileSize = MAX_ZIP_FILE_SIZE): """ Preparing response for downloading zip-file, consist of all files in folder (without recursion!) Content-Disposition header field from http://tools.ietf.org/html/rfc2183 :param: folder - instance of Folder model :return: HttpResponse with zip file and some parameters Message in case of zip file overflow max size """ zipSubdir = folder.name zipFilename = "%s.zip" % folder.name sio = BytesIO() # Open StringIO to grab in-memory ZIP contents zipFile = zipfile.ZipFile(sio, "w") # The zip compressor zipFileSize = 0 msg = "" for report in Report.objects.filter(parent_id=folder.id): zipFileSize += report.file.size if zipFileSize > maxFileSize: msg = 'Завеликий zip. Решту файлів відкинуто' break filename = report.filename # "людська" назва файла filepath = report.file.name # шлях до файла на диску abs_path = os.path.join(MEDIA_ROOT, filepath) zipPath = os.path.join(zipSubdir, filename) # шлях в архіві zipFile.write(abs_path, zipPath) # add file to zip zipFile.close() # Must close zip for all contents to be written fileExt = ".zip" ct = get_mimeType().get(fileExt.lower(), "application/octet-stream") fn = ' filename="%s";' % transliterate(zipFilename) fns = " filename*=utf-8''%s;" % urlquote(zipFilename) # Grab ZIP file from in-memory, make response with correct MIME-type response = HttpResponse(sio.getvalue(), content_type=ct) response['Content-Disposition'] = 'attachment' + fn + fns response['Content-Length'] = len(sio.getbuffer()) return response, zipFilename, msg
def assertBlock(self, python, java): self.maxDiff = None dump = False py_block = PyBlock(parent=PyModule('test', 'test.py')) if python: python = adjust(python) code = compile(python, '<test>', 'exec') py_block.extract(code, debug=dump) java_code = py_block.transpile() out = BytesIO() constant_pool = ConstantPool() java_code.resolve(constant_pool) constant_pool.add(Utf8('test')) constant_pool.add(Utf8('Code')) constant_pool.add(Utf8('LineNumberTable')) writer = ClassFileWriter(out, constant_pool) java_code.write(writer) debug = StringIO() reader = ClassFileReader(BytesIO(out.getbuffer()), constant_pool, debug=debug) JavaCode.read(reader, dump=0) if dump: print(debug.getvalue()) java = adjust(java) self.assertEqual(debug.getvalue(), java[1:])
def __create_small_picture__(request): """ Generates a smaller, standard size (128x128) picture of original image with filename <o_o_image_filename>. Filename of smaller file is <o_o_image_filename>_sm If this fails, the smaller file will we removed! :param request: the originial request object :param o_image: the filename of original image :return: True on success, False else """ profile = request.user.profile outfile = profile.profile_picture_full.path + "_sm" try: im = Image.open(request.user.profile.profile_picture_full.path) im.thumbnail((128, 128)) thumb_io = BytesIO() im.save(thumb_io, format='JPEG') thumb_file = InMemoryUploadedFile(thumb_io, None, 'pp.jpg', 'image/jpeg', thumb_io.getbuffer().nbytes, None) profile.profile_picture_small = thumb_file profile.save() return True except IOError: logging.error("Fehler beim speichern des thumbnails") try: os.remove(outfile) except IOError: pass return False
def save(self, *args, **kwargs): if self.pk is None: saved_picture = self.picture self.picture = None super().save(*args, **kwargs) self.picture = saved_picture # TODO-працює збереження файлів після зміни розміру. # Змінити: # - зберігати файл 200х200 під тою ж назвою 1.jpg # - в template фільтр 200х200 прибрати, а дозволити показувати # все зображення (бо воно вже на етапі збереження # обрізане до 200*200) if self.picture: # Якщо з форми не приходить новий файл, то self.picture.name # міститиме шлях до старого файла, який відповідатиме шаблону # "profile_images/[0-9]*.jpg". # У протилежному випадку self.picture.name - це новий файл, # який потрібно мінімізувати перед збереженням. if not CheckPathMatchMediaPattern(self.picture.name): image = Image.open(BytesIO(self.picture.read())) image.thumbnail((200,200), Image.ANTIALIAS) output = BytesIO() image.save(output, format='JPEG', quality=90, optimize=1) output.seek(0) self.picture= InMemoryUploadedFile(output,'ImageField', self.picture.name, 'image/jpeg', output.getbuffer().__len__(), None) super().save(*args, **kwargs)
def crop_img(image, crop): try: temp = Image.open(image) format = temp.format # Rotate image based off of exif data (used by mobile devices that automatically orients the picture) # http://www.lifl.fr/~damien.riquet/auto-rotating-pictures-using-pil.html try: exif = temp._getexif() orientation_key = 274 if exif: if orientation_key in exif: orientation = exif[orientation_key] rotate_values = {3: 180, 6: 270, 8: 90} if orientation in rotate_values: temp = temp.rotate(rotate_values[orientation], expand=1) except Exception: pass extra = json.loads(crop) x = int(extra.get('x')) y = int(extra.get('y')) height = int(extra.get('height')) width = int(extra.get('width')) temp = temp.crop((x, y, x + width, y + height)) # Django imagefield is a InMemoryUploadedFile, need to convert PIL image in order to save temp_io = BytesIO() temp.save(temp_io, format=format) return InMemoryUploadedFile(temp_io, None, image.name, 'image/' + format, temp_io.getbuffer().nbytes, None) except Exception: print(traceback.format_exc()) return 'Unable to read image, try a different image'
def main(): content = BytesIO(b'Hello again') key = b'32byteslongsecretkeymustprovided' encryption_key = base64.b64encode(key).decode() encryption_key_md5 = base64.b64encode(hashlib.md5(key).digest()).decode() minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey) # Put object with special headers which encrypt object in S3 with provided key minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, metadata={ 'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': encryption_key, 'x-amz-server-side-encryption-customer-key-MD5': encryption_key_md5 }) # Get decrypted object with same headers obj = minio.get_object(STORAGE_BUCKET, 'test_crypt1.txt', request_headers={ 'x-amz-server-side-encryption-customer-algorithm': 'AES256', 'x-amz-server-side-encryption-customer-key': encryption_key, 'x-amz-server-side-encryption-customer-key-MD5': encryption_key_md5 }) print(obj.read())
def decompress(fd): window = array('B', (b' ' * 4078) + (b'\x00' * 18)) pos = 4078 _out = BytesIO() def out(byte): nonlocal window, pos window[pos] = byte pos = (pos + 1) & 0xFFF _out.write(bytes([byte])) try: while True: for encoded in bits(fd.read(1)[0]): if encoded: out(fd.read(1)[0]) else: #encoded code = fd.read(2) offset = code[0] | (code[1] & 0xF0) << 4 length = (code[1] & 0xF) + 3 for x in range(offset, offset+length): out(window[x & 0xFFF]) except IndexError: pass return _out.getbuffer()
class ContainerWriter(object): def __init__(self, fp, schema, sync_marker=None): self.writer = Writer(schema) self.fp = fp self.sync_marker = sync_marker or os.urandom(16) self.header_written = sync_marker is not None self.records = 0 self.buffer = BytesIO() def write_header(self): assert not self.header_written, "Header is already written once" Writer(HEADER_SCHEMA).write(self.fp, { "magic": b"Obj\x01", "meta": { "avro.schema": json.dumps(self.schema.json).encode("utf8"), "avro.codec": b"null" }, "sync": self.sync_marker }) self.header_written = True def write(self, message): self.writer.write(self.buffer, message) self.records += 1 if self.buffer.tell() > 1024 ** 2: self.flush() def flush(self): if not self.header_written: self.write_header() self.header_written = True if not self.records: return write_long(self.fp, self.records) write_long(self.fp, self.buffer.tell()) self.fp.write(self.buffer.getbuffer()) self.fp.write(self.sync_marker) self.fp.flush() self.records = 0 self.buffer = BytesIO() @property def schema(self): """Returns the :class:`avrolight.schema.Schema` instance that this writer uses.""" return self.writer.schema def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.flush() close = flush
def generate_preview(self, new_ver=False): if new_ver: b = BytesIO() self._get_workspace().save(b, "png") return b.getbuffer().tobytes() else: return self._magic.dump(mode="preview")
def post_meshes_to_dvid(config, instance_name, partition_items): """ Send the given meshes (either .obj or .drc) as key/value pairs to DVID. Args: config: The CreateMeshes workflow config data instance_name: key-value instance to post to partition_items: tuple (group_id, [(segment_id, mesh_data), (segment_id, mesh_data)]) """ # Re-use session for connection pooling. session = default_dvid_session() # Re-use resource manager client connections, too. # (If resource-server is empty, this will return a "dummy client") resource_client = ResourceManagerClient( config["options"]["resource-server"], config["options"]["resource-port"] ) dvid_server = config["dvid-info"]["dvid"]["server"] uuid = config["dvid-info"]["dvid"]["uuid"] grouping_scheme = config["mesh-config"]["storage"]["grouping-scheme"] mesh_format = config["mesh-config"]["storage"]["format"] if grouping_scheme == "no-groups": for group_id, segment_ids_and_meshes in partition_items: for (segment_id, mesh_data) in segment_ids_and_meshes: @auto_retry(3, pause_between_tries=60.0, logging_name=__name__) def write_mesh(): with resource_client.access_context(dvid_server, False, 2, len(mesh_data)): session.post(f'{dvid_server}/api/node/{uuid}/{instance_name}/key/{segment_id}', mesh_data) session.post(f'{dvid_server}/api/node/{uuid}/{instance_name}/key/{segment_id}_info', json={ 'format': mesh_format }) write_mesh() else: # All other grouping schemes, including 'singletons' write tarballs. # (In the 'singletons' case, there is just one tarball per body.) for group_id, segment_ids_and_meshes in partition_items: tar_name = _get_group_name(config, group_id) tar_stream = BytesIO() with closing(tarfile.open(tar_name, 'w', tar_stream)) as tf: for (segment_id, mesh_data) in segment_ids_and_meshes: mesh_name = _get_mesh_name(config, segment_id) f_info = tarfile.TarInfo(mesh_name) f_info.size = len(mesh_data) tf.addfile(f_info, BytesIO(mesh_data)) tar_bytes = tar_stream.getbuffer() @auto_retry(3, pause_between_tries=60.0, logging_name=__name__) def write_tar(): with resource_client.access_context(dvid_server, False, 1, len(tar_bytes)): session.post(f'{dvid_server}/api/node/{uuid}/{instance_name}/key/{tar_name}', tar_bytes) write_tar()
def genJiaozhunCS(c,fn): tree = ET.parse(fn) root = tree.getroot() Worksheets=root.findall("{urn:schemas-microsoft-com:office:spreadsheet}Worksheet") Worksheet = Worksheets[0] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") setCell(table,10,8,c.yiqixinghao)#setCell(table,10,8).Value =c.yiqixinghao setCell(table,16,8,c.yiqibh)# setCell(table,16,8).Value =c.yiqibh setCell(table,18,8,c.yonghu)# setCell(table,18,8).Value =c.yonghu setCell(table,20,8,c.addr)# setCell(table,18,8).Value =c.yonghu d=c.yujifahuo_date d1=d+datetime.timedelta(-1) setCell(table,32,8,str(d1.year))# setCell(table,32,8).FormulaR1C1 =d1.year#年 setCell(table,32,12,str(d1.month))# setCell(table,32,12).Value =d1.month#月 setCellWithFont(table,32,16,[str(d1.day)])# setCell(table,32,16).Value =d1.day#日 d2=d+datetime.timedelta(364) setCell(table,35,8,str(d2.year))# setCell(table,35,8).Value =d2.year#年 setCell(table,35,12,str(d2.month))# setCell(table,35,12).Value =d2.month#月 setCellWithFont(table,35,16,[str(d2.day)])# setCell(table,35,16).Value =d2.day#日 # #page 2 Worksheet = Worksheets[1] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") #dd=" 地点(LOCATION): "+c.yonghu setCellWithFont(table,26,3,["地点","(","LOCATION"+")"+":"+" "+c.yonghu])# setCell(table,22,3).Value=ddsetCell(table,26,3,dd)# setCell(table,26,3,dd # #page 3 Worksheet = Worksheets[2] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") eles=[] for i in range(6): eles.append(getCell(table,13,8+i*2))#setCell(table,13,8+i*2).Value) stds=[] for i in range(6): stds.append(getCell(table,14,8+i*2))#setCell(table,14,8+i*2).Value) (tests,errs)=genTest(eles,stds) for i in range(6): setCell(table,15,8+i*2,tests[i])#setCell(table,15,8+i*2).Value=tests[i] for i in range(6): setCell(table,16,8+i*2,errs[i])#setCell(table,16,8+i*2).Value=errs[i] # #jmd cave=0.0725 crsd=0.3/100 (rs,ave_str,rsd_str)=genjmd(cave,crsd) cjmd_str=",".join(rs) setCell(table,20,3," 测量值(C/%):" +cjmd_str)# setCell(table,20,3).Value=" 测量值(C/%):" +cjmd_str setCell(table,21,3," 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%"))# setCell(table,21,3).Value=" 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%") sjmd_str=" 0.0721,0.0725,0.0723,0.0727,0.0730,0.0725,0.0725" save=0.0723 srsd=1.04/100 (rs,ave_str,rsd_str)=genjmd(save,srsd) sjmd_str=",".join(rs) setCell(table,22,3," 测量值(S/%):" + sjmd_str)# setCell(table,22,3).Value=" 测量值(S/%):" + sjmd_str setCell(table,23,3," 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%"))# setCell(table,23,3).Value=" 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%") s=BytesIO() tree.write(s, encoding="utf-8", xml_declaration=True, method="xml") s.seek(0) data=s.getbuffer() return data
def send_fig(fig, name): """ sends figure to frontend """ imgdata = BytesIO() fig.savefig(imgdata, format='png') imgdata.seek(0) # rewind the data uri = 'data:image/png;base64,' + urllib.parse.quote( base64.encodebytes(imgdata.getbuffer())) send_action("update_plot", params={"src": uri, "name": name})
class MessageProtocol(asyncio.Protocol): def __init__(self): self.len_buff = bytearray(4) self.len_offset = 0 self.msg_len = None self.msg_data = BytesIO() self.msg_offset = 0 self.transport = None def close(self): transport = self.transport if transport: # Calling transport.close() from an event handler seems to # cause errors, even if from a completely unrelated connection # (but in the same loop). transport._loop.call_soon(transport.close) def connection_made(self, transport): self.transport = transport def connection_lost(self, exc): self.transport = None def send_msg(self, msg): if self.transport: self.transport.writelines(msg_to_bytes(msg)) def data_received(self, data): data_offset = 0 while data_offset != len(data): if not self.msg_len: remaining = len(self.len_buff) - self.len_offset to_add = min(remaining, len(data) - data_offset) self.len_buff[self.len_offset:self.len_offset + to_add] = \ data[data_offset:data_offset + to_add] self.len_offset += to_add data_offset += to_add if to_add == remaining: (self.msg_len,) = struct.unpack('!I', self.len_buff) self.len_offset = 0 self.msg_data.seek(0) else: remaining = self.msg_len - self.msg_data.tell() to_add = min(remaining, len(data) - data_offset) self.msg_data.write(data[data_offset:data_offset + to_add]) data_offset += to_add if to_add == remaining: self.process_msg(tuple(msg_from_bytes(self.msg_data.getbuffer()[:self.msg_len]))) assert sys.getrefcount(self.msg_data) == 2, "never store message references!" self.msg_len = None def process_msg(self, msg): raise NotImplementedError()
def genJiaozhunN(c,fn): tree = ET.parse(fn) root = tree.getroot() Worksheets=root.findall("{urn:schemas-microsoft-com:office:spreadsheet}Worksheet") Worksheet = Worksheets[0] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") setCell(table,10,8,c.yiqixinghao) setCell(table,16,8,c.yiqibh) setCell(table,18,8,c.yonghu) setCell(table,20,8,c.addr)# setCell(table,18,8).Value =c.yonghu d=c.yujifahuo_date d1=d+datetime.timedelta(-1) setCell(table,32,8,str(d1.year))#年 setCell(table,32,12,str(d1.month))#月 setCell(table,32,16,str(d1.day))#日 d2=d+datetime.timedelta(364) setCell(table,35,8,str(d2.year))#年 setCell(table,35,12,str(d2.month))#月 setCell(table,35,16,str(d2.day))#日 #page 2 Worksheet = Worksheets[1] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") dd=" 地点(LOCATION): "+c.yonghu setCellWithFont(table,22,3,["地点","(","LOCATION",")",":"," ",c.yonghu])# setCell(table,22,3).Value=dd#setCell(table,22,3,dd) #page 3 Worksheet = Worksheets[2] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") eles=[] for i in [11,18]: eles.append(getCell(table,13,i)) stds=[] for i in [11,18]: stds.append(getCell(table,14,i)) print(eles,stds) (tests,errs)=genTestR(eles,stds) tmp=0 for i in [11,18]: setCell(table,15,i,tests[tmp]) tmp +=1 tmp=0 for i in [11,18]: setCell(table,16,i,errs[tmp]) tmp +=1 #jmd save=0.0084 srsd=1.5/100 (rs,ave_str,rsd_str)=genjmd(save,srsd) sjmd_str=",".join(rs) setCell(table,22,3," 测量值(N/%):" + sjmd_str) setCell(table,23,3," 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%")) s=BytesIO() tree.write(s, encoding="utf-8", xml_declaration=True, method="xml") s.seek(0) data=s.getbuffer() return data
def get_package(self, name): # if isinstance(source, types.ModuleType): # source = os.path.dirname(source.__file__) if name: target = BytesIO() try: zipapp.create_archive('wheels/' + name, target=target) except zipapp.ZipAppError: pass else: # TODO: hash and cache? return target.getbuffer()
def write_image(file_path, image): file, extension = path.splitext(file_path) if extension.lower() == ".webp": logging.error("writing .webp %s", os.path.abspath(file_path)) fp = BytesIO() im_pil = Image.fromarray(image) im_pil.save(fp, Image.registered_extensions()['.webp']) storage.save(file_path, ContentFile(fp.getbuffer())) else: logging.info("writing jpeg %s %s", file_path, extension) encoded, image = cv2.imencode(extension, image) storage.save(file_path, ContentFile(image))
async def get(url): loop = asyncio.get_event_loop() buffer = BytesIO() id = url[35:].replace('/', " ") start = time.time() Logger.info("Fetching {0}".format(id)) for i in range(ATTEMPTS): try: res = await loop.run_in_executor(None, lambda: requests.get(url, stream=True)) if res.status_code == 200: for chunk in res.iter_content(DEFAULT_BUFFER_SIZE): buffer.write(chunk) Logger.info("Fetched {0} completed in {1}s".format(id, time.time() - start)) if len(buffer.getbuffer()) <= 0: Logger.info("Buffer for {0} is empty ".format(id)) return buffer.getbuffer() else: Logger.warn("Request to {0} failed with error code : {1} ".format(url, str(res.status_code))) except Exception as e: Logger.warn("Request {0} failed with exception : {1}".format(id, str(e))) time.sleep(0.5 * i) raise Exception("Request failed for {0} after ATTEMPTS attempts".format(url))
def genJiaozhunO(c,fn): tree = ET.parse(fn) root = tree.getroot() Worksheets=root.findall("{urn:schemas-microsoft-com:office:spreadsheet}Worksheet") Worksheet = Worksheets[0] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") setCell(table,16,8,c.yiqibh)#setCell(table,10,8).Value =c.yiqixinghao setCell(table,11,8).Value =c.yiqixinghao setCell(table,18,8,c.yonghu)#setCell(table,18,8).Value =c.yonghu setCell(table,20,8,c.addr)# setCell(table,18,8).Value =c.yonghu d=c.yujifahuo_date d1=d+datetime.timedelta(-1) setCell(table,32,8,str(d1.year))#setCell(table,32,8).FormulaR1C1 =d1.year#年 setCell(table,32,12,str(d1.month))#setCell(table,32,12).Value =d1.month#月 setCell(table,32,16,str(d1.day))#setCell(table,32,16).Value =d1.day#日 d2=d+datetime.timedelta(364) setCell(table,35,8,str(d2.year))#setCell(table,35,8).Value =d2.year#年 setCell(table,35,12,str(d2.month))#setCell(table,35,12).Value =d2.month#月 setCell(table,35,16,str(d2.day))#setCell(table,35,16).Value =d2.day#日 # #page 2 Worksheet = Worksheets[1] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") #dd=" 地点(LOCATION): "+c.yonghu setCellWithFont(table,22,3,["地点","(","LOCATION",")",":"," ",c.yonghu])# setCell(table,22,3).Value=dd # #page 3 Worksheet = Worksheets[2]# w = a.Worksheets[2] table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") eles=[] for i in [8,15]: print("getCell",13,i) eles.append(getCell(table,13,i))#.Value) stds=[] for i in [8,15]: stds.append(getCell(table,14,i))#setCell(table,14,i).Value) (tests,errs)=genTestR(eles,stds) setCell(table,15,8,tests[0]) # setCell(table,15,8).Value=tests[0] setCell(table,15,15,tests[1]) # setCell(table,15,15).Value=tests[1] setCell(table,16,8,errs[0]) # setCell(table,16,8).Value=errs[0] setCell(table,16,15,errs[1]) # setCell(table,16,15).Value=errs[1] # #jmd cave=0.0134 crsd=0.74/100 (rs,ave_str,rsd_str)=genjmd(cave,crsd) cjmd_str=",".join(rs) setCell(table,20,3," 测量值(O/%):" +cjmd_str) # setCell(table,20,3).Value=" 测量值(O/%):" +cjmd_str setCell(table,21,3," 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%")) # setCell(table,21,3).Value=" 平均值:%s, 相对标准偏差:%s" % (ave_str+"%",rsd_str+"%") s=BytesIO() tree.write(s, encoding="utf-8", xml_declaration=True, method="xml") s.seek(0) data=s.getbuffer() return data
def png_buffer(array): """Convert an array to PNG, handling transparency in an at-least-partially-sane manner.""" assert array.ndim == 2 im = toimage(array) alpha = toimage(array != 0) im.putalpha(alpha) # Return format is a buffer of PNG-encoded data fp = BytesIO() im.save(fp, format='png') return fp.getbuffer()
def redll(buf, mapping): # mapping is a dict of bytes->bytes for dlls to rename # Make the section to store the new strings. section_offset_mapping = {} data_writer = BytesIO() for old_dll, new_dll in mapping.items(): # section_offset_mapping contains lowercased DLL names old_dll = old_dll.decode("ascii").lower().encode("ascii") data_offset = data_writer.tell() data_writer.write(new_dll) data_writer.write(b"\x00") section_offset_mapping[old_dll] = data_offset data = data_writer.getbuffer() # I checked python27.dll, and its DLL name RVAs point into section .rdata # which has characteristics: # 0x40000040 # = 0x40000000 IMAGE_SCN_MEM_READ # + 40 IMAGE_SCN_CNT_INITIALIZED_DATA # which sounds about right. characteristics = IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA new_buf, new_section_rva = add_section(buf, data, characteristics) rva_mapping = {old_dll: new_section_rva + offset for (old_dll, offset) in section_offset_mapping.items()} pe_headers = view_pe_headers(new_buf) unused_names = set(rva_mapping) def rewrite_names(viewer, name_field): for s in viewer(pe_headers): name_offset = rva_to_file_offset(pe_headers.sections, s[name_field]) name, _ = read_asciiz(new_buf, name_offset) # lowercase name for case-insensitive matching name = name.decode("ascii").lower().encode("ascii") if name in rva_mapping: # print("RVA: %s -> %s" # % (hex(s[name_field]), hex(rva_mapping[name]))) s[name_field] = rva_mapping[name] unused_names.discard(name) rewrite_names(view_import_directory_tables, "Name RVA") rewrite_names(view_delay_load_directory_tables, "Name") if unused_names: warnings.warn(UserWarning, "Did not find any imports from following DLLs: " + ", ".join(str(name) for name in unused_names)) return new_buf
def test_write_jpeg(): """See if Pillow can write JPEG (tests linkage against mozjpeg)""" im = Image.new('RGB', (10, 10)) buffer = BytesIO() im.save(buffer, format='JPEG') if sys.version_info[0] == 2: buffer.seek(0) size = len(buffer.read()) else: size = buffer.getbuffer().nbytes if size != 375: logger.error("JPEG optimization is not working as expected! size=%s", size)
def main(): minio = Minio(STORAGE_ENDPOINT, access_key=AWSAccessKeyId, secret_key=AWSSecretKey) content = BytesIO(b'Hello again') #Create an SSE_S3 object sse_s3_obj = SSE_S3() # Put object with from SSE_S3 object which encrypt object in S3 with provided key minio.put_object(STORAGE_BUCKET, 'test_crypt.txt', content, content.getbuffer().nbytes, sse=sse_s3_obj) # Get decrypted object with same headers obj = minio.get_object(STORAGE_BUCKET, 'test_crypt.txt') print(obj.read())
class ColorPreview(): def __init__(self, color="#000000"): self.img = Image.new('RGB', (100, 100), color); self.stream = BytesIO(); self.img.save(self.stream, format="PNG"); self.stream.seek(0, 0); def getBuffer(self): return self.stream.getbuffer(); def getStream(self): return self.stream; def getValue(self): return self.stream.getvalue();
def fetch_sram(ctx, sram_file): """Fetch a SRAM dump from the Super Wild Card.""" with serial.Serial(ctx['com_port'], timeout=3) as ser: start_time = time() ser.write(pack('>10sx', b'READ SRAM')) sram = BytesIO() with click.progressbar(range(0, BLOCK_SIZE * 4), label='Receiving') as all_bytes: for _ in all_bytes: byte = ser.read(size=1) if byte == b'': raise click.ClickException('Transfer timeout.') sram.write(byte) trail = ser.read(size=9) if trail != b'*#*#*#*OK': raise click.ClickException('Transfer failed! ({})'.format(trail)) sram_file.write(sram.getbuffer()) click.echo(click.style('Transfer complete in {0:.2f} seconds.'.format(time() - start_time), fg='green'))
def genShujubiao(c,fn): tree = ET.parse(fn) root = tree.getroot() Worksheet=root.find("{urn:schemas-microsoft-com:office:spreadsheet}Worksheet") table=Worksheet.find("{urn:schemas-microsoft-com:office:spreadsheet}Table") if c.yiqixinghao=="CS-2800": setCell(table,4,3,getCell(table,4,3)+"√") #setCell(table,4,3).Value =setCell(table,4,3).Value +"√" setCell(table,8,3,"√") setCell(table,8,5,"√") setCell(table,8,6,"√") elif c.yiqixinghao=="CS-3000": setCell(table,4,4,getCell(table,4,4)+"√") #setCell(table,4,3).Value =setCell(table,4,3).Value +"√" setCell(table,8,3,"√") setCell(table,8,5,"√") setCell(table,8,6,"√") elif c.yiqixinghao=="O-3000": setCell(table,4,5,getCell(table,4,5)+"√")#setCell(table,4,5).Value =setCell(table,4,5).Value +"√" setCell(table,11,3,"√")#setCell(table,11,3).Value ="√" elif c.yiqixinghao=="N-3000": setCell(table,4,6,getCell(table,4,6)+"√")#setCell(table,4,6).Value =setCell(table,4,6).Value +"√" elif c.yiqixinghao=="ON-3000": setCell(table,5,3,getCell(table,5,3)+"√")#setCell(table,5,3).Value =setCell(table,5,3).Value +"√" setCell(table,11,3,"√")#setCell(table,11,3).Value ="√" setCell(table,11,4,"√")#setCell(table,11,4).Value ="√" elif c.yiqixinghao=="ONH-3000": setCell(table,5,4,getCell(table,5,4)+"√")#setCell(table,5,4).Value =setCell(table,5,4).Value +"√" setCell(table,11,3,"√")#setCell(table,11,3).Value ="√" setCell(table,11,4,"√")#setCell(table,11,4).Value ="√" setCell(table,11,5,"√")#setCell(table,11,3).Value ="√" elif c.yiqixinghao=="OH-3000": setCell(table,5,6,getCell(table,5,6)+"√")#setCell(table,5,6).Value =setCell(table,5,6).Value +"√" setCell(table,11,3,"√")#setCell(table,11,3).Value ="√" setCell(table,11,5,"√")#setCell(table,11,5).Value ="√" setCell(table,2,6,"合同号:"+c.hetongbh)#setCell(table,2,6).Value ="合同号:"+c.hetongbh setCell(table,6,3,c.yiqibh) #setCell(table,6,3).Value =c.yiqibh setCell(table,3,3,c.yonghu) #setCell(table,3,3).Value =c.yonghu d=datetime.datetime.now() setCell(table,13,4,str(d.year)+"年") #setCell(table,13,4).FormulaR1C1 =str(d.year)+"年" setCell(table,13,5,str(d.month)+"月")#setCell(table,13,5).Value =str(d.month)+"月" setCell(table,13,6,str(d.day)+"日")#setCell(table,13,6).Value =str(d.day)+"日" setCell(table,14,3,c.baoxiang)#setCell(table,13,6).Value =str(d.day)+"日" s=BytesIO() tree.write(s, encoding="utf-8", xml_declaration=True, method="xml") s.seek(0) data=s.getbuffer() return data
def scan(self, step=None): if step: self.current_step = step logger.debug('Do scan %d/%d' % (self.current_step, self.steps)) phi = self.current_step * 2 * pi / self.steps sample_n = 100 r_l = 40 r_r = 50 # x = sin(theta) * cos(phi) * r # y = sin(theta) * sin(phi) * r # z = r * cos(theta) point_L = [] point_R = [] for t in range(sample_n): theta = t * 2 * pi / sample_n x = sin(theta) * cos(phi) y = sin(theta) * sin(phi) z = cos(theta) point_L.append([x * r_l, y * r_l, z * r_l + r_l / 2, 0, 0, 0]) point_R.append([x * r_r, y * r_r, z * r_r + r_r / 2, 255, 0, 0]) # point_L # point_R buf = BytesIO() def packer(s, points): for p in points: s.write( struct.pack('<ffffff', p[0], p[1], p[2], p[3] / 255., p[4] / 255., p[5] / 255.)) packer(buf, point_L) packer(buf, point_R) self.send_text('{"status": "chunk", "left": %d, "right": %d}' % (len(point_L), len(point_R))) self.send_binary(buf.getbuffer()) # self.task.forward() # from time import sleep # sleep(1) self.send_ok() self.current_step += 1
def serialize(data): """ :param data: :return: """ try: bio = BytesIO() pickle.dump(data, bio, protocol=pickle.HIGHEST_PROTOCOL) try: # noinspection PyUnresolvedReferences pickled_data = bio.getbuffer() except AttributeError: pickled_data = bio.getvalue() except ImportError: pickled_data = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL) return pickled_data
def render_and_send_app(pk): app = import_module('venueapp.models').Application.objects.get(pk=pk) cover = import_module('venueapp.views').make_cover_page(app) max_bytes = config.get_int("max_inline_attachment_bytes", 0) for venue in app.venues.all(): html = render_to_string( "venueapp/pdf_app.html", { "object": app, "cover": cover, "venue": venue, "logo": finders.find("logo.png"), "pdf": True, "max_attachment_size": max_bytes, }) doc = HTML(string=html, base_url=settings.SITE_URL).render() bookmark_tree = doc.make_bookmark_tree() bookmarks = list([_Bookmark(i) for i in bookmark_tree]) app_pdf = BytesIO() doc.write_pdf(app_pdf) merger = PdfFileMerger() merger.append(app_pdf, import_bookmarks=False) for staff in app.staffmember_set.signed_on().filter( role__accepts_attachment=True).exclude( Q(attachment=None) | Q(attachment="")): name = "{} {}'s ".format(staff.role_name, staff.person) try: if staff.attachment.size < max_bytes: reader = PdfFileReader(staff.attachment.open(), False) attachment_pages = reader.getNumPages() page = None for i, bookmark in enumerate(bookmarks): if bookmark.label == name + "Supplement": page = bookmarks[i + 1].location if page: merger.merge(page, staff.attachment.open(), import_bookmarks=False) for i in bookmarks: if i.location >= page: i.location += attachment_pages else: merger.append(staff.attachment.open(), bookmark=name + "Attachment", import_bookmarks=False) except Exception as e: tb.print_exc() for i in bookmarks: merger.addBookmark(i.label, i.location) pdf = BytesIO() # open("/tmp/{}.pdf".format(venue.venue), "wb") merger.write(pdf) msg = render_msg("venueapp/email/submission.html", locals(), to=[ "{} <{}>".format(i.get_full_name(False), i.email) for i in venue.managers.all() ], cc=[ "{} <{}>".format(i.get_full_name(False), i.email) for i in app.show.staff.all() ], subject="Application for {} in {} Submitted".format( app, venue.venue), tags=["venueapp", "venueapp-submission"]) msg.attach("{} - {}.pdf".format(app, venue), BytesIO(pdf.getbuffer()), "application/pdf") try: msg.send() except Exception as err: LOGGER.error("Application submission sending failed: {}".format( repr(err))) tb.print_exc() finally: merger.close()
def t_time(thDate): data_dic = thDate try: # 로그파일 불러오기 df = pd.read_csv("tempHumid-" + data_dic + ".log", names=['Datetime', 'etc'], header=None, index_col='Datetime') # 인덱스를 datetimeindex로 바꾸기 df.index = pd.to_datetime(df.index) # etc 컬럼 나누기 df[['etc1', 'etc2', 'etc3', 'Temp', 'Humidity']] = \ df['etc'].str.split(' ', n=5, expand=True) # Temp 컬럼에서 숫자만 빼오기, str->numeric df['Temp'] = df['Temp'].str.slice(start=5, stop=-1) df['Temp'] = df['Temp'].apply(pd.to_numeric) # Generate the figure **without using pyplot**. fig, ax = plt.subplots() # 습도를 그래프로 그림, 선색 지정 ax.plot(df['Temp'], color='#ff0303') # 그래프 타이틀 title = '{}.{}.{}'.format(df.index.year[0], df.index.month[0], df.index.day[0])\ +'\'s temperature\n'\ +'──────────────────────\n'\ +'Highest: {}℃ / '.format(df['Temp'].max())\ +'Lowest: {}℃'.format(df['Temp'].min()) font = {'size': 20} plt.title(title, fontdict=font) # x축, y축 라벨 ax.set_xlabel('Time', size=16) ax.set_ylabel('Temperature(℃)', size=16) # 주석 달았을 때 그래프 밖으로 삐져 나오는 것 방지 ax.spines['top'].set_color('none') ax.spines['right'].set_color('none') # x축 간격 지정, 5분 hours = mdates.HourLocator(interval=1) h_fmt = mdates.DateFormatter('%H:%M:%S') ax.xaxis.set_major_locator(hours) ax.xaxis.set_major_formatter(h_fmt) # Show the major grid lines plt.grid(b=True, which='major', color='#b3b3b3', linestyle='-') # # Show the minor grid lines # plt.minorticks_on() # plt.grid(b=True, which='minor', color='#a6a6a6', linestyle='--', alpha=0.1) # 최고 온도일 때 주석 # df_cond_max = df[df['Temp'] == df['Temp'].max()] # df_max = df_cond_max.loc[:, ['Temp']] # max_idx = mpl.dates.date2num(df_max.index.to_pydatetime()) # 최고 온도가 2개 이상일 때 처리 # arrowprops = dict(arrowstyle="->") # for i in range(0, len(max_idx)): # plt.annotate('{}'.format(df['Temp'].max()),\ # xy=(max_idx[i], df['Temp'].max()),\ # xytext=(max_idx[i]+0.0005, df['Temp'].max()),\ # horizontalalignment='left', verticalalignment='top', color='#154a31' # ) # 최저 온도일 때 주석 # df_cond_min = df[df['Temp'] == df['Temp'].min()] # df_min = df_cond_min.loc[:, ['Temp']] # min_idx = mpl.dates.date2num(df_min.index.to_pydatetime()) # 최저 온도가 2개 이상일 때 처리 # for i in range(0, len(min_idx)): # plt.annotate('{}'.format(df['Temp'].min()),\ # xy=(min_idx[i], df['Temp'].min()),\ # xytext=(min_idx[i]+0.0005, df['Temp'].min()),\ # horizontalalignment='left', verticalalignment='top', color='#154a31', # ) # Save it to a temporary buffer. buf = BytesIO() fig.set_size_inches(12, 9) fig.savefig(buf, format="png", dpi=70) # Embed the result in the html output. data = base64.b64encode(buf.getbuffer()).decode("ascii") return f"<img src='data:image/png;base64,{data}'/>" except: return "Error"
def _download_results(self, result_dict): results = ResultFiles() if 'metadata' in result_dict: if 'reference' in result_dict['metadata']: try: logger.debug("Downloading CTS/VTS log from: %s" % result_dict['metadata']['reference']) self.tradefed_results_url = result_dict['metadata'][ 'reference'] result_tarball_request = requests.get( self.tradefed_results_url) if result_tarball_request.status_code == 200: result_tarball_request.raw.decode_content = True r = BytesIO(result_tarball_request.content) results.tradefed_zipfile = ExtractedResult() results.tradefed_zipfile.contents = r results.tradefed_zipfile.length = len( result_tarball_request.content) results.tradefed_zipfile.name = result_tarball_request.url.rsplit( "/", 1)[1] results.tradefed_zipfile.mimetype = result_tarball_request.headers.get( "Content-Type") logger.debug("Retrieved %s bytes" % r.getbuffer().nbytes) t = tarfile.open(fileobj=r, mode='r:xz') for member in t.getmembers(): logger.debug("Available member: %s" % member.name) if "test_result.xml" in member.name: results.test_results = self._extract_member( t, member) logger.debug( "test_results object is empty: %s" % (results.test_results is None)) if "compatibility_result.xsl" in member.name: results.test_result_xslt = self._extract_member( t, member) logger.debug( "test_result_xslt object is empty: %s" % (results.test_result_xslt is None)) if "compatibility_result.css" in member.name: results.test_result_css = self._extract_member( t, member) logger.debug( "test_result_css object is empty: %s" % (results.test_result_css is None)) if "logo.png" in member.name: results.test_result_image = self._extract_member( t, member) logger.debug( "test_result_image object is empty: %s" % (results.test_result_image is None)) if "tradefed-stdout.txt" in member.name: results.tradefed_stdout = self._extract_member( t, member) logger.debug( "tradefed_stdout object is empty: %s" % (results.tradefed_stdout is None)) if "tradefed-logcat.txt" in member.name: results.tradefed_logcat = self._extract_member( t, member) logger.debug( "tradefed_logcat object is empty: %s" % (results.tradefed_logcat is None)) except tarfile.TarError as e: logger.warning(e) except EOFError as e: # this can happen when tarfile is corrupted logger.warning(e) except requests.exceptions.Timeout as e: logger.warning(e) return results
def setUp(self): self.maxDiff = None conn = engine.connect() text = (f"INSERT INTO projects (uuid, name, created_at, updated_at) " f"VALUES (%s, %s, %s, %s)") conn.execute(text, ( PROJECT_ID, NAME, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s, %s)") conn.execute(text, ( EXPERIMENT_ID, NAME, PROJECT_ID, POSITION, 1, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s, %s)") conn.execute(text, ( EXPERIMENT_ID_2, NAME, PROJECT_ID, POSITION, 1, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s, %s)") conn.execute(text, ( EXPERIMENT_ID_3, NAME, PROJECT_ID, POSITION, 1, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO tasks (uuid, name, description, image, commands, arguments, tags, parameters, experiment_notebook_path, deployment_notebook_path, is_default, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)") conn.execute(text, ( TASK_ID, NAME, DESCRIPTION, IMAGE, COMMANDS_JSON, ARGUMENTS_JSON, TAGS_JSON, dumps([]), EXPERIMENT_NOTEBOOK_PATH, DEPLOYMENT_NOTEBOOK_PATH, 0, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s)") conn.execute(text, ( OPERATOR_ID, EXPERIMENT_ID, TASK_ID, PARAMETERS_JSON, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s)") conn.execute(text, ( OPERATOR_ID_2, EXPERIMENT_ID_2, TASK_ID, PARAMETERS_JSON_2, CREATED_AT, UPDATED_AT, )) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at) " f"VALUES (%s, %s, %s, %s, %s, %s)") conn.execute(text, ( OPERATOR_ID_3, EXPERIMENT_ID_3, TASK_ID, PARAMETERS_JSON_3, CREATED_AT, UPDATED_AT, )) conn.close() # uploads mock dataset try: MINIO_CLIENT.make_bucket(BUCKET_NAME) except BucketAlreadyOwnedByYou: pass file = BytesIO((b'col0,col1,col2,col3,col4,col5\n' b'01/01/2000,5.1,3.5,1.4,0.2,Iris-setosa\n' b'01/01/2000,5.1,3.5,1.4,0.2,Iris-setosa\n' b'01/01/2000,5.1,3.5,1.4,0.2,Iris-setosa\n')) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=f"datasets/{DATASET}/{DATASET}", data=file, length=file.getbuffer().nbytes, ) metadata = { "columns": ["col0", "col1", "col2", "col3", "col4", "col5"], "featuretypes": [ DATETIME, NUMERICAL, NUMERICAL, NUMERICAL, NUMERICAL, CATEGORICAL ], "filename": DATASET, "run_id": RUN_ID, } buffer = BytesIO(dumps(metadata).encode()) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=f"datasets/{DATASET}/{DATASET}.metadata", data=buffer, length=buffer.getbuffer().nbytes, ) file = BytesIO((b'foo,bar,baz,qux\n' b'01/01/2000,foo,1.2,2.3\n' b'01/01/2000,bar,2.3,3.4\n' b'01/01/2000,baz,4.5,4.5\n')) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=f"datasets/{DATASET_2}/{DATASET_2}", data=file, length=file.getbuffer().nbytes, ) metadata = { "columns": ["foo", "bar", "baz", "qux"], "featuretypes": [DATETIME, CATEGORICAL, NUMERICAL, NUMERICAL], "filename": DATASET_2, "runId": None } buffer = BytesIO(dumps(metadata).encode()) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=f"datasets/{DATASET_2}/{DATASET_2}.metadata", data=buffer, length=buffer.getbuffer().nbytes, ) MINIO_CLIENT.copy_object( bucket_name=BUCKET_NAME, object_name= f"datasets/{DATASET}/runs/{RUN_ID}/operators/{OPERATOR_ID}/{DATASET}/{DATASET}", object_source=f"/{BUCKET_NAME}/datasets/{DATASET}/{DATASET}", ) MINIO_CLIENT.copy_object( bucket_name=BUCKET_NAME, object_name= f"datasets/{DATASET}/runs/{RUN_ID}/operators/{OPERATOR_ID}/{DATASET}/{DATASET}.metadata", object_source= f"/{BUCKET_NAME}/datasets/{DATASET}/{DATASET}.metadata", )
def handler(event, context): # Load settings SETTINGS = BossSettings.load() # Used as a guard against trying to delete the SQS message when lambda is # triggered by SQS. sqs_triggered = 'Records' in event and len(event['Records']) > 0 if sqs_triggered : # Lambda invoked by an SQS trigger. msg_data = json.loads(event['Records'][0]['body']) # Load the project info from the chunk key you are processing chunk_key = msg_data['chunk_key'] proj_info = BossIngestProj.fromSupercuboidKey(chunk_key) proj_info.job_id = msg_data['ingest_job'] else: # Standard async invoke of this lambda. # Load the project info from the chunk key you are processing proj_info = BossIngestProj.fromSupercuboidKey(event["chunk_key"]) proj_info.job_id = event["ingest_job"] # Get message from SQS ingest queue, try for ~2 seconds rx_cnt = 0 msg_data = None msg_id = None msg_rx_handle = None while rx_cnt < 6: ingest_queue = IngestQueue(proj_info) try: msg = [x for x in ingest_queue.receiveMessage()] # StopIteration may be converted to a RunTimeError. except (StopIteration, RuntimeError): msg = None if msg: msg = msg[0] print("MESSAGE: {}".format(msg)) print(len(msg)) msg_id = msg[0] msg_rx_handle = msg[1] msg_data = json.loads(msg[2]) print("MESSAGE DATA: {}".format(msg_data)) break else: rx_cnt += 1 print("No message found. Try {} of 6".format(rx_cnt)) time.sleep(1) if not msg_id: # No tiles ready to ingest. print("No ingest message available") return # Get the chunk key of the tiles to ingest. chunk_key = msg_data['chunk_key'] tile_error_queue = TileErrorQueue(proj_info) print("Ingesting Chunk {}".format(chunk_key)) tiles_in_chunk = int(chunk_key.split('&')[1]) # Setup SPDB instance sp = SpatialDB(msg_data['parameters']["KVIO_SETTINGS"], msg_data['parameters']["STATEIO_CONFIG"], msg_data['parameters']["OBJECTIO_CONFIG"]) # Get tile list from Tile Index Table tile_index_db = BossTileIndexDB(proj_info.project_name) # tile_index_result (dict): keys are S3 object keys of the tiles comprising the chunk. tile_index_result = tile_index_db.getCuboid(msg_data["chunk_key"], int(msg_data["ingest_job"])) if tile_index_result is None: # If chunk_key is gone, another lambda uploaded the cuboids and deleted the chunk_key afterwards. if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to chunk key missing from tile index table") return # Sort the tile keys print("Tile Keys: {}".format(tile_index_result["tile_uploaded_map"])) tile_key_list = [x.rsplit("&", 2) for x in tile_index_result["tile_uploaded_map"].keys()] if len(tile_key_list) < tiles_in_chunk: print("Not a full set of 16 tiles. Assuming it has handled already, tiles: {}".format(len(tile_key_list))) if not sqs_triggered: ingest_queue.deleteMessage(msg_id, msg_rx_handle) return tile_key_list = sorted(tile_key_list, key=lambda x: int(x[1])) tile_key_list = ["&".join(x) for x in tile_key_list] print("Sorted Tile Keys: {}".format(tile_key_list)) # Augment Resource JSON data so it will instantiate properly that was pruned due to S3 metadata size limits resource_dict = msg_data['parameters']['resource'] _, exp_name, ch_name = resource_dict["boss_key"].split("&") resource_dict["channel"]["name"] = ch_name resource_dict["channel"]["description"] = "" resource_dict["channel"]["sources"] = [] resource_dict["channel"]["related"] = [] resource_dict["channel"]["default_time_sample"] = 0 resource_dict["channel"]["downsample_status"] = "NOT_DOWNSAMPLED" resource_dict["experiment"]["name"] = exp_name resource_dict["experiment"]["description"] = "" resource_dict["experiment"]["num_time_samples"] = 1 resource_dict["experiment"]["time_step"] = None resource_dict["experiment"]["time_step_unit"] = None resource_dict["coord_frame"]["name"] = "cf" resource_dict["coord_frame"]["name"] = "" resource_dict["coord_frame"]["x_start"] = 0 resource_dict["coord_frame"]["x_stop"] = 100000 resource_dict["coord_frame"]["y_start"] = 0 resource_dict["coord_frame"]["y_stop"] = 100000 resource_dict["coord_frame"]["z_start"] = 0 resource_dict["coord_frame"]["z_stop"] = 100000 resource_dict["coord_frame"]["voxel_unit"] = "nanometers" # Setup the resource resource = BossResourceBasic() resource.from_dict(resource_dict) dtype = resource.get_numpy_data_type() # read all tiles from bucket into a slab tile_bucket = TileBucket(proj_info.project_name) data = [] num_z_slices = 0 for tile_key in tile_key_list: try: image_data, message_id, receipt_handle, metadata = tile_bucket.getObjectByKey(tile_key) except KeyError: print('Key: {} not found in tile bucket, assuming redelivered SQS message and aborting.'.format( tile_key)) if not sqs_triggered: # Remove message so it's not redelivered. ingest_queue.deleteMessage(msg_id, msg_rx_handle) print("Aborting due to missing tile in bucket") return image_bytes = BytesIO(image_data) image_size = image_bytes.getbuffer().nbytes # Get tiles size from metadata, need to shape black tile if actual tile is corrupt. if 'x_size' in metadata: tile_size_x = metadata['x_size'] else: print('MetadataMissing: x_size not in tile metadata: using 1024.') tile_size_x = 1024 if 'y_size' in metadata: tile_size_y = metadata['y_size'] else: print('MetadataMissing: y_size not in tile metadata: using 1024.') tile_size_y = 1024 if image_size == 0: print('TileError: Zero length tile, using black instead: {}'.format(tile_key)) error_msg = 'Zero length tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) else: try: # DP NOTE: Issues when specifying dtype in the asarray function with Pillow ver 8.3.1. # Fixed by separating array instantiation and dtype assignment. tile_img = np.asarray(Image.open(image_bytes)) tile_img = tile_img.astype(dtype) except TypeError as te: print('TileError: Incomplete tile, using black instead (tile_size_in_bytes, tile_key): {}, {}' .format(image_size, tile_key)) error_msg = 'Incomplete tile' enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) except OSError as oe: print('TileError: OSError, using black instead (tile_size_in_bytes, tile_key): {}, {} ErrorMessage: {}' .format(image_size, tile_key, oe)) error_msg = 'OSError: {}'.format(oe) enqueue_tile_error(tile_error_queue, tile_key, chunk_key, error_msg) tile_img = np.zeros((tile_size_x, tile_size_y), dtype=dtype) data.append(tile_img) num_z_slices += 1 # Make 3D array of image data. It should be in XYZ at this point chunk_data = np.array(data) del data tile_dims = chunk_data.shape # Break into Cube instances print("Tile Dims: {}".format(tile_dims)) print("Num Z Slices: {}".format(num_z_slices)) num_x_cuboids = int(math.ceil(tile_dims[2] / CUBOIDSIZE[proj_info.resolution][0])) num_y_cuboids = int(math.ceil(tile_dims[1] / CUBOIDSIZE[proj_info.resolution][1])) print("Num X Cuboids: {}".format(num_x_cuboids)) print("Num Y Cuboids: {}".format(num_y_cuboids)) chunk_key_parts = BossUtil.decode_chunk_key(chunk_key) t_index = chunk_key_parts['t_index'] for x_idx in range(0, num_x_cuboids): for y_idx in range(0, num_y_cuboids): # TODO: check time series support cube = Cube.create_cube(resource, CUBOIDSIZE[proj_info.resolution]) cube.zeros() # Compute Morton ID # TODO: verify Morton indices correct! print(chunk_key_parts) morton_x_ind = x_idx + (chunk_key_parts["x_index"] * num_x_cuboids) morton_y_ind = y_idx + (chunk_key_parts["y_index"] * num_y_cuboids) print("Morton X: {}".format(morton_x_ind)) print("Morton Y: {}".format(morton_y_ind)) morton_index = XYZMorton([morton_x_ind, morton_y_ind, int(chunk_key_parts['z_index'])]) # Insert sub-region from chunk_data into cuboid x_start = x_idx * CUBOIDSIZE[proj_info.resolution][0] x_end = x_start + CUBOIDSIZE[proj_info.resolution][0] x_end = min(x_end, tile_dims[2]) y_start = y_idx * CUBOIDSIZE[proj_info.resolution][1] y_end = y_start + CUBOIDSIZE[proj_info.resolution][1] y_end = min(y_end, tile_dims[1]) z_end = CUBOIDSIZE[proj_info.resolution][2] # TODO: get sub-array w/o making a copy. print("Yrange: {}".format(y_end - y_start)) print("Xrange: {}".format(x_end - x_start)) print("X start: {}".format(x_start)) print("X stop: {}".format(x_end)) cube.data[0, 0:num_z_slices, 0:(y_end - y_start), 0:(x_end - x_start)] = chunk_data[0:num_z_slices, y_start:y_end, x_start:x_end] # Create object key object_key = sp.objectio.generate_object_key(resource, proj_info.resolution, t_index, morton_index) print("Object Key: {}".format(object_key)) # Put object in S3 sp.objectio.put_objects([object_key], [cube.to_blosc()]) # Add object to index sp.objectio.add_cuboid_to_index(object_key, ingest_job=int(msg_data["ingest_job"])) # Update id indices if this is an annotation channel # We no longer index during ingest. #if resource.data['channel']['type'] == 'annotation': # try: # sp.objectio.update_id_indices( # resource, proj_info.resolution, [object_key], [cube.data]) # except SpdbError as ex: # sns_client = boto3.client('sns') # topic_arn = msg_data['parameters']["OBJECTIO_CONFIG"]["prod_mailing_list"] # msg = 'During ingest:\n{}\nCollection: {}\nExperiment: {}\n Channel: {}\n'.format( # ex.message, # resource.data['collection']['name'], # resource.data['experiment']['name'], # resource.data['channel']['name']) # sns_client.publish( # TopicArn=topic_arn, # Subject='Object services misuse', # Message=msg) lambda_client = boto3.client('lambda', region_name=SETTINGS.REGION_NAME) names = AWSNames.from_lambda(context.function_name) delete_tiles_data = { 'tile_key_list': tile_key_list, 'region': SETTINGS.REGION_NAME, 'bucket': tile_bucket.bucket.name } # Delete tiles from tile bucket. lambda_client.invoke( FunctionName=names.delete_tile_objs.lambda_, InvocationType='Event', Payload=json.dumps(delete_tiles_data).encode() ) delete_tile_entry_data = { 'tile_index': tile_index_db.table.name, 'region': SETTINGS.REGION_NAME, 'chunk_key': chunk_key, 'task_id': msg_data['ingest_job'] } # Delete entry from tile index. lambda_client.invoke( FunctionName=names.delete_tile_index_entry.lambda_, InvocationType='Event', Payload=json.dumps(delete_tile_entry_data).encode() ) if not sqs_triggered: # Delete message since it was processed successfully ingest_queue.deleteMessage(msg_id, msg_rx_handle)
def _binary_to_bytesio(binary: io.BytesIO): if isinstance(binary, io.BytesIO): return binary.getbuffer() return io.BytesIO(binary).getbuffer()
def setUp(self): self.maxDiff = None conn = engine.connect() text = ( f"INSERT INTO tasks (uuid, name, description, image, commands, arguments, tags, experiment_notebook_path, deployment_notebook_path, is_default, created_at, updated_at) " f"VALUES ('{TASK_ID}', '{NAME}', '{DESCRIPTION}', '{IMAGE}', '{COMMANDS_JSON}', '{ARGUMENTS_JSON}', '{TAGS_JSON}', '{EXPERIMENT_NOTEBOOK_PATH}', '{DEPLOYMENT_NOTEBOOK_PATH}', 0, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) text = ( f"INSERT INTO tasks (uuid, name, description, image, commands, arguments, tags, experiment_notebook_path, deployment_notebook_path, is_default, created_at, updated_at) " f"VALUES ('{TASK_ID_2}', 'foo 2', '{DESCRIPTION}', '{IMAGE}', '{COMMANDS_JSON}', '{ARGUMENTS_JSON}', '{TAGS_JSON}', '{EXPERIMENT_NOTEBOOK_PATH_2}', '{DEPLOYMENT_NOTEBOOK_PATH_2}', 0, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) conn.close() try: MINIO_CLIENT.make_bucket(BUCKET_NAME) except BucketAlreadyOwnedByYou: pass file = BytesIO(SAMPLE_NOTEBOOK.encode("utf-8")) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=EXPERIMENT_NOTEBOOK_PATH[len(f"minio://{BUCKET_NAME}/" ):], data=file, length=file.getbuffer().nbytes, ) file = BytesIO( b'{"cells":[{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":4}' ) MINIO_CLIENT.put_object( bucket_name=BUCKET_NAME, object_name=DEPLOYMENT_NOTEBOOK_PATH[len(f"minio://{BUCKET_NAME}/" ):], data=file, length=file.getbuffer().nbytes, ) session = requests.Session() session.cookies.update(COOKIES) session.headers.update(HEADERS) session.hooks = { "response": lambda r, *args, **kwargs: r.raise_for_status(), } session.put( url=f"{JUPYTER_ENDPOINT}/api/contents/tasks", data=dumps({ "type": "directory", "content": None }), ) session.put( url=f"{JUPYTER_ENDPOINT}/api/contents/tasks/{NAME}", data=dumps({ "type": "directory", "content": None }), ) session.put( url= f"{JUPYTER_ENDPOINT}/api/contents/tasks/{NAME}/Deployment.ipynb", data=dumps({ "type": "notebook", "content": loads(SAMPLE_NOTEBOOK) }), ) session.put( url= f"{JUPYTER_ENDPOINT}/api/contents/tasks/{NAME}/Experiment.ipynb", data=dumps({ "type": "notebook", "content": loads(SAMPLE_NOTEBOOK) }), )
write_uint32(f, texsection_size) f.seek(texdata_end) f.write(b"DNOS") sound_size_offset = f.tell() f.write(b"FOOO") write_uint32(f, len(resinfo["Level name"])) f.write(bytes(resinfo["Level name"], encoding="ascii")) f.write(b"HFSB") write_uint32(f, 4) write_uint32(f, len(sounds) // 2) for entry in sounds: #f.write(entry.data) entry.write(f) end = f.tell() sound_section_size = end - sound_size_offset - 4 f.seek(sound_size_offset) write_uint32(f, sound_section_size) f.seek(end) for entry in itertools.chain(models, animations, effects, scripts): #f.write(entry.data) entry.write(f) with bwopen(output, "wb") as final: final.write(f.getbuffer())
def test_file_flow(file_service, location, example_file_record, identity_simple): """Test the lifecycle of a file. - Initialize file saving - Save 3 files - Commit the files - List files of the record - Read file metadata - Retrieve a file - Delete a file - Delete all remaining files - List should be empty """ recid = example_file_record['id'] file_to_initialise = [{ 'key': 'article.txt', 'checksum': 'md5:c785060c866796cc2a1708c997154c8e', 'size': 17, # 2kB 'metadata': { 'description': 'Published article PDF.', } }] # Initialize file saving result = file_service.init_files(recid, identity_simple, file_to_initialise) assert result.to_dict()['entries'][0]['key'] == \ file_to_initialise[0]['key'] # # Save 3 files # to_files = ['one', 'two', 'three'] # for to_file in to_files: content = BytesIO(b'test file content') result = file_service.set_file_content(recid, file_to_initialise[0]['key'], identity_simple, content, content.getbuffer().nbytes) # TODO figure response for succesfully saved file assert result.to_dict()['key'] == file_to_initialise[0]['key'] result = file_service.commit_file(recid, 'article.txt', identity_simple) # TODO currently there is no status in the json between the initialisation # and the commiting. assert result.to_dict()['key'] == \ file_to_initialise[0]['key'] # List files result = file_service.list_files(recid, identity_simple) assert result.to_dict()['entries'][0]['key'] == \ file_to_initialise[0]['key'] # Read file metadata result = file_service.read_file_metadata(recid, 'article.txt', identity_simple) assert result.to_dict()['key'] == \ file_to_initialise[0]['key'] # Retrieve file result = file_service.get_file_content(recid, 'article.txt', identity_simple) assert result.file_id == 'article.txt' # Delete file result = file_service.delete_file(recid, 'article.txt', identity_simple, 0) assert result.file_id == 'article.txt' # Assert deleted result = file_service.list_files(recid, identity_simple) assert result.files assert not result.files.get('article.txt') # Delete all remaining files result = file_service.delete_all_files(recid, identity_simple) assert result.files == {} # Assert deleted result = file_service.list_files(recid, identity_simple) assert result.files == {}
def write_file(buffer: io.BytesIO, uri: str, buffer_size: int = io.DEFAULT_BUFFER_SIZE) -> None: with open_buffered_stream_writer(uri, buffer_size=buffer_size) as output_stream: output_stream.write(buffer.getbuffer())
class Packer(object): """ MessagePack Packer Usage:: packer = Packer() astream.write(packer.pack(a)) astream.write(packer.pack(b)) Packer's constructor has some keyword arguments: :param callable default: Convert user type to builtin type that Packer supports. See also simplejson's document. :param bool use_single_float: Use single precision float type for float. (default: False) :param bool autoreset: Reset buffer after each pack and return its content as `bytes`. (default: True). If set this to false, use `bytes()` to get content and `.reset()` to clear buffer. :param bool use_bin_type: Use bin type introduced in msgpack spec 2.0 for bytes. It also enables str8 type for unicode. (default: True) :param bool strict_types: If set to true, types will be checked to be exact. Derived classes from serializable types will not be serialized and will be treated as unsupported type and forwarded to default. Additionally tuples will not be serialized as lists. This is useful when trying to implement accurate serialization for python types. :param bool datetime: If set to true, datetime with tzinfo is packed into Timestamp type. Note that the tzinfo is stripped in the timestamp. You can get UTC datetime with `timestamp=3` option of the Unpacker. (Python 2 is not supported). :param str unicode_errors: The error handler for encoding unicode. (default: 'strict') DO NOT USE THIS!! This option is kept for very specific usage. Example of streaming deserialize from file-like object:: unpacker = Unpacker(file_like) for o in unpacker: process(o) Example of streaming deserialize from socket:: unpacker = Unpacker() while True: buf = sock.recv(1024**2) if not buf: break unpacker.feed(buf) for o in unpacker: process(o) Raises ``ExtraData`` when *packed* contains extra bytes. Raises ``OutOfData`` when *packed* is incomplete. Raises ``FormatError`` when *packed* is not valid msgpack. Raises ``StackError`` when *packed* contains too nested. Other exceptions can be raised during unpacking. """ def __init__( self, default=None, use_single_float=False, autoreset=True, use_bin_type=True, strict_types=False, datetime=False, unicode_errors=None, ): self._strict_types = strict_types self._use_float = use_single_float self._autoreset = autoreset self._use_bin_type = use_bin_type self._buffer = StringIO() if PY2 and datetime: raise ValueError("datetime is not supported in Python 2") self._datetime = bool(datetime) self._unicode_errors = unicode_errors or "strict" if default is not None: if not callable(default): raise TypeError("default must be callable") self._default = default def _pack( self, obj, nest_limit=DEFAULT_RECURSE_LIMIT, check=isinstance, check_type_strict=_check_type_strict, ): default_used = False if self._strict_types: check = check_type_strict list_types = list else: list_types = (list, tuple) while True: if nest_limit < 0: raise ValueError("recursion limit exceeded") if obj is None: return self._buffer.write(b"\xc0") if check(obj, bool): if obj: return self._buffer.write(b"\xc3") return self._buffer.write(b"\xc2") if check(obj, int_types): if 0 <= obj < 0x80: return self._buffer.write(struct.pack("B", obj)) if -0x20 <= obj < 0: return self._buffer.write(struct.pack("b", obj)) if 0x80 <= obj <= 0xFF: return self._buffer.write(struct.pack("BB", 0xCC, obj)) if -0x80 <= obj < 0: return self._buffer.write(struct.pack(">Bb", 0xD0, obj)) if 0xFF < obj <= 0xFFFF: return self._buffer.write(struct.pack(">BH", 0xCD, obj)) if -0x8000 <= obj < -0x80: return self._buffer.write(struct.pack(">Bh", 0xD1, obj)) if 0xFFFF < obj <= 0xFFFFFFFF: return self._buffer.write(struct.pack(">BI", 0xCE, obj)) if -0x80000000 <= obj < -0x8000: return self._buffer.write(struct.pack(">Bi", 0xD2, obj)) if 0xFFFFFFFF < obj <= 0xFFFFFFFFFFFFFFFF: return self._buffer.write(struct.pack(">BQ", 0xCF, obj)) if -0x8000000000000000 <= obj < -0x80000000: return self._buffer.write(struct.pack(">Bq", 0xD3, obj)) if not default_used and self._default is not None: obj = self._default(obj) default_used = True continue raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) if n >= 2**32: raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, unicode): obj = obj.encode("utf-8", self._unicode_errors) n = len(obj) if n >= 2**32: raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize if n >= 2**32: raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, float): if self._use_float: return self._buffer.write(struct.pack(">Bf", 0xCA, obj)) return self._buffer.write(struct.pack(">Bd", 0xCB, obj)) if check(obj, (ExtType, Timestamp)): if check(obj, Timestamp): code = -1 data = obj.to_bytes() else: code = obj.code data = obj.data assert isinstance(code, int) assert isinstance(data, bytes) L = len(data) if L == 1: self._buffer.write(b"\xd4") elif L == 2: self._buffer.write(b"\xd5") elif L == 4: self._buffer.write(b"\xd6") elif L == 8: self._buffer.write(b"\xd7") elif L == 16: self._buffer.write(b"\xd8") elif L <= 0xFF: self._buffer.write(struct.pack(">BB", 0xC7, L)) elif L <= 0xFFFF: self._buffer.write(struct.pack(">BH", 0xC8, L)) else: self._buffer.write(struct.pack(">BI", 0xC9, L)) self._buffer.write(struct.pack("b", code)) self._buffer.write(data) return if check(obj, list_types): n = len(obj) self._pack_array_header(n) for i in xrange(n): self._pack(obj[i], nest_limit - 1) return if check(obj, dict): return self._pack_map_pairs(len(obj), dict_iteritems(obj), nest_limit - 1) if self._datetime and check(obj, _DateTime) and obj.tzinfo is not None: obj = Timestamp.from_datetime(obj) default_used = 1 continue if not default_used and self._default is not None: obj = self._default(obj) default_used = 1 continue raise TypeError("Cannot serialize %r" % (obj, )) def pack(self, obj): try: self._pack(obj) except: self._buffer = StringIO() # force reset raise if self._autoreset: ret = self._buffer.getvalue() self._buffer = StringIO() return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) if self._autoreset: ret = self._buffer.getvalue() self._buffer = StringIO() return ret def pack_array_header(self, n): if n >= 2**32: raise ValueError self._pack_array_header(n) if self._autoreset: ret = self._buffer.getvalue() self._buffer = StringIO() return ret def pack_map_header(self, n): if n >= 2**32: raise ValueError self._pack_map_header(n) if self._autoreset: ret = self._buffer.getvalue() self._buffer = StringIO() return ret def pack_ext_type(self, typecode, data): if not isinstance(typecode, int): raise TypeError("typecode must have int type.") if not 0 <= typecode <= 127: raise ValueError("typecode should be 0-127") if not isinstance(data, bytes): raise TypeError("data must have bytes type") L = len(data) if L > 0xFFFFFFFF: raise ValueError("Too large data") if L == 1: self._buffer.write(b"\xd4") elif L == 2: self._buffer.write(b"\xd5") elif L == 4: self._buffer.write(b"\xd6") elif L == 8: self._buffer.write(b"\xd7") elif L == 16: self._buffer.write(b"\xd8") elif L <= 0xFF: self._buffer.write(b"\xc7" + struct.pack("B", L)) elif L <= 0xFFFF: self._buffer.write(b"\xc8" + struct.pack(">H", L)) else: self._buffer.write(b"\xc9" + struct.pack(">I", L)) self._buffer.write(struct.pack("B", typecode)) self._buffer.write(data) def _pack_array_header(self, n): if n <= 0x0F: return self._buffer.write(struct.pack("B", 0x90 + n)) if n <= 0xFFFF: return self._buffer.write(struct.pack(">BH", 0xDC, n)) if n <= 0xFFFFFFFF: return self._buffer.write(struct.pack(">BI", 0xDD, n)) raise ValueError("Array is too large") def _pack_map_header(self, n): if n <= 0x0F: return self._buffer.write(struct.pack("B", 0x80 + n)) if n <= 0xFFFF: return self._buffer.write(struct.pack(">BH", 0xDE, n)) if n <= 0xFFFFFFFF: return self._buffer.write(struct.pack(">BI", 0xDF, n)) raise ValueError("Dict is too large") def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) for (k, v) in pairs: self._pack(k, nest_limit - 1) self._pack(v, nest_limit - 1) def _pack_raw_header(self, n): if n <= 0x1F: self._buffer.write(struct.pack("B", 0xA0 + n)) elif self._use_bin_type and n <= 0xFF: self._buffer.write(struct.pack(">BB", 0xD9, n)) elif n <= 0xFFFF: self._buffer.write(struct.pack(">BH", 0xDA, n)) elif n <= 0xFFFFFFFF: self._buffer.write(struct.pack(">BI", 0xDB, n)) else: raise ValueError("Raw is too large") def _pack_bin_header(self, n): if not self._use_bin_type: return self._pack_raw_header(n) elif n <= 0xFF: return self._buffer.write(struct.pack(">BB", 0xC4, n)) elif n <= 0xFFFF: return self._buffer.write(struct.pack(">BH", 0xC5, n)) elif n <= 0xFFFFFFFF: return self._buffer.write(struct.pack(">BI", 0xC6, n)) else: raise ValueError("Bin is too large") def bytes(self): """Return internal buffer contents as bytes object""" return self._buffer.getvalue() def reset(self): """Reset internal buffer. This method is useful only when autoreset=False. """ self._buffer = StringIO() def getbuffer(self): """Return view of internal buffer.""" if USING_STRINGBUILDER or PY2: return memoryview(self.bytes()) else: return self._buffer.getbuffer()
def mock_imgur5k(tmpdir_factory, mock_image_stream): file = BytesIO(mock_image_stream) image_folder = tmpdir_factory.mktemp("images") label_folder = tmpdir_factory.mktemp("dataset_info") labels = { "index_id": { "YsaVkzl": { "image_url": "https://i.imgur.com/YsaVkzl.jpg", "image_path": "/path/to/IMGUR5K-Handwriting-Dataset/images/YsaVkzl.jpg", "image_hash": "993a7cbb04a7c854d1d841b065948369" }, "wz3wHhN": { "image_url": "https://i.imgur.com/wz3wHhN.jpg", "image_path": "/path/to/IMGUR5K-Handwriting-Dataset/images/wz3wHhN.jpg", "image_hash": "9157426a98ee52f3e1e8d41fa3a99175" }, "BRHSP23": { "image_url": "https://i.imgur.com/BRHSP23.jpg", "image_path": "/path/to/IMGUR5K-Handwriting-Dataset/images/BRHSP23.jpg", "image_hash": "aab01f7ac82ae53845b01674e9e34167" } }, "index_to_ann_map": { "YsaVkzl": ["YsaVkzl_0", "YsaVkzl_1", "YsaVkzl_2"], "wz3wHhN": ["wz3wHhN_0", "wz3wHhN_1"], "BRHSP23": ["BRHSP23_0"] }, "ann_id": { "YsaVkzl_0": { "word": "I", "bounding_box": "[605.33, 1150.67, 614.33, 226.33, 81.0]" }, "YsaVkzl_1": { "word": "am", "bounding_box": "[783.67, 654.67, 521.0, 222.33, 56.67]" }, "YsaVkzl_2": { "word": "a", "bounding_box": "[959.0, 437.0, 76.67, 201.0, 38.33]" }, "wz3wHhN_0": { "word": "jedi", "bounding_box": "[783.67, 654.67, 521.0, 222.33, 56.67]" }, "wz3wHhN_1": { "word": "!", "bounding_box": "[959.0, 437.0, 76.67, 201.0, 38.33]" }, "BRHSP23_0": { "word": "jedi", "bounding_box": "[783.67, 654.67, 521.0, 222.33, 56.67]" } } } label_file = label_folder.join("imgur5k_annotations.json") with open(label_file, 'w') as f: json.dump(labels, f) for index_id in ['YsaVkzl', 'wz3wHhN', 'BRHSP23']: fn_i = image_folder.join(f"{index_id}.jpg") with open(fn_i, 'wb') as f: f.write(file.getbuffer()) return str(image_folder), str(label_file)
def decode(data: bytes, max_size: int = 4096) -> bytearray: """ Performs LZSS decoding Parameters ---------- data: bytes A string of bytes to decompress max_size: int Maximum size of uncompressed data, in bytes Returns ------- bytearray A bytearray containing the uncompressed data """ reader = BytesIO(data) length = len(reader.getbuffer()) flags = 0 # Encoded flag flags_used = 7 # Unencoded flag out_data = bytearray() while len(out_data) < max_size: flags = flags >> 1 flags_used = flags_used + 1 # If all flag bits have been shifted out, read a new flag if flags_used == 8: if reader.tell() == length: break flags = reader.read(1)[0] flags_used = 0 # Found an unencoded byte if (flags & 1) != 0: if reader.tell() == length: break out_data.append(reader.read(1)[0]) # Found encoded data else: if reader.tell() == length: break code_offset = reader.read(1)[0] if reader.tell() == length: break code_length = reader.read(1)[0] + MAX_UNENCODED + 1 for i in range(0, code_length): out_data.append(out_data[len(out_data) - (code_offset + 1)]) return out_data
def encode(data: any) -> bytes: """ Perform LZSS Algorithm Encoding """ global hash_table writer = BytesIO() input_buffer = bytearray(data) length = len(input_buffer) if length == 0: return bytes(0) # Start with an empty list hash_table = list() for i in range(0, HASH_SIZE): hash_table.append([]) flag_data = FlagData() # 8 code flags and 8 encoded strings flag_data.flags = 0 flag_data.flag_position = 1 encoded_data = bytearray(256 * 8) flag_data.next_encoded = 0 # Next index of encoded data input_buffer_position = 0 # Head of encoded lookahead for i in range(0, length - MAX_UNENCODED): hash_key = get_hash_key(input_buffer, i) hash_table[hash_key].append(i) match_data = find_match(input_buffer, input_buffer_position) while input_buffer_position < length: # Extend match length if trailing rubbish is present if input_buffer_position + match_data.length > length: match_data.length = length - input_buffer_position # Write unencoded byte if match is not long enough if match_data.length <= MAX_UNENCODED: match_data.length = 1 # 1 unencoded byte flag_data.flags = flag_data.flags | flag_data.flag_position # Flag unencoded byte encoded_data[ flag_data.next_encoded] = input_buffer[input_buffer_position] flag_data.next_encoded = flag_data.next_encoded + 1 update_flags(flag_data, encoded_data, writer) # Encode as offset and length if match length >= max unencoded else: match_data.offset = (input_buffer_position - 1) - match_data.offset if match_data.offset > 255 or match_data.offset < 0: print("Match Data Offset out of range!") return bytes(0) if match_data.length - (MAX_UNENCODED + 1) > 255: print("Match Data Length out of range!") return bytes(0) encoded_data[flag_data.next_encoded] = match_data.offset flag_data.next_encoded = flag_data.next_encoded + 1 encoded_data[flag_data.next_encoded] = match_data.length - ( MAX_UNENCODED + 1) flag_data.next_encoded = flag_data.next_encoded + 1 update_flags(flag_data, encoded_data, writer) input_buffer_position = input_buffer_position + match_data.length # Find next match match_data = find_match(input_buffer, input_buffer_position) # Write any remaining encoded data if flag_data.next_encoded != 0: writer.write(flag_data.flags.to_bytes(1, "little")) for i in range(0, flag_data.next_encoded): writer.write(encoded_data[i].to_bytes(1, "little")) return writer.getbuffer().tobytes(order='A')
def generate_plot(self, plot_size=6): plot_size = int(self.plot_size) if int(self.plot_size) else plot_size if len(self.point) == 2: My_X, My_Y = np.meshgrid(range(10 * plot_size), range(10 * plot_size)) My_X = (My_X - 5 * plot_size) / 5 My_Y = (My_Y - 5 * plot_size) / 5 My_Z = np.ones((10 * plot_size, 10 * plot_size)) for i in range(10 * plot_size): for j in range(10 * plot_size): My_Z[i, j] = self.calculate_function_value_in_point( self.function, [My_X[i, j], My_Y[i][0]]) fig = Figure() ax = Axes3D(fig) # 1stplot X, Y, Z = My_X, My_Y, My_Z ax.plot_surface(X, Y, Z, rstride=8, cstride=8, alpha=0.3) ax.plot3D(np.array(self.path_x), np.array(self.path_y), np.array(self.path_z), c='r', marker='o') cset = ax.contour(X, Y, Z, zdir='z', offset=-100, cmap=cm.coolwarm) cset = ax.contour(X, Y, Z, zdir='x', offset=-40, cmap=cm.coolwarm) cset = ax.contour(X, Y, Z, zdir='y', offset=40, cmap=cm.coolwarm) buf = BytesIO() fig.savefig(buf, format="png") img1 = base64.b64encode(buf.getbuffer()).decode("ascii") # 2nd plot ax.view_init(azim=-90, elev=90) buf = BytesIO() fig.savefig(buf, format="png") img2 = base64.b64encode(buf.getbuffer()).decode("ascii") # 3rd plot surf = ax.plot_surface(My_X, My_Y, My_Z, rstride=1, cstride=1, cmap=cm.coolwarm, linewidth=0, antialiased=False) ax.plot(np.array(self.path_x), np.array(self.path_y), np.array(self.path_z), 'ro', alpha=0.5) ax.zaxis.set_major_locator(LinearLocator(10)) ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f')) fig.colorbar(surf, shrink=0.5, aspect=5) ax.view_init(azim=None, elev=None) buf = BytesIO() fig.savefig(buf, format="png") img3 = base64.b64encode(buf.getbuffer()).decode("ascii") img = [img1, img2, img3] return img #return plt #plt.show() else: return [ "It's not 3D object", "It's not 3D object", "It's not 3D object" ]
def pie_city(): #cities = ['ha hoi', 'thanh pho ho chi minh', 'da nang'] db = get_db() jobs_count = db.jobs_info.count() cities = db.jobs_info.distinct('city') city_job_data = [] c = [] num = [] salary = [] for city in cities: query = {"city": city} # print(city + '-' + str(db.jobs_info.find(query).count()/jobs_count)) # if(db.jobs_info.find(query).count()/jobs_count > 0.05): city_job_data.append({ 'city': city, 'num': db.jobs_info.find(query).count(), }) for job in db.jobs_info.find(): if (job["maxSalary"] != 0): salary.append(job["maxSalary"] / 1000000) # print(job["maxSalary"]) sorted_data = sorted(city_job_data, key=lambda k: k['num'], reverse=True) for i in range(5): c.append(sorted_data[i]['city']) num.append(sorted_data[i]['num']) # Draw Chart # print(city_jobs_count) fig = Figure() gs = fig.add_gridspec(1, 1) ax1 = fig.add_subplot(gs[0, 0]) ax1.set_title("5 tỉnh thành có số lương công việc nhiều nhất") # ax2 = fig.add_subplot(gs[1, 0]) # ax3 = fig.add_subplot(gs[0, 1]) # ax4 = fig.add_subplot(gs[1, 1]) pie = ax1.pie(num, autopct='%1.3f%%') ax1.legend(pie[0], c, loc="lower right", bbox_to_anchor=(0.25, 0)) # ax2.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # ax3.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # ax4.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # Save it to a temporary buffer. buf = BytesIO() fig.savefig(buf, format="png") # Embed the result in the html output. data = base64.b64encode(buf.getbuffer()).decode("ascii") # return f"<img src='data:image/png;base64,{data}'/>" fig2 = Figure() gs2 = fig2.add_gridspec(1, 1) ax2 = fig2.add_subplot(gs[0, 0]) ax2.set_xlabel('Mức lương (triệu đồng)') ax2.set_ylabel('Số lượng công việc') ax2.set_title('Phân bố mức lương') # ax2 =fig2.add_subplot(gs[1, 0]) # ax3 = fig2.add_subplot(gs[0, 1]) # ax4 = fig2.add_subplot(gs[1, 1]) hist = ax2.hist(salary, 80) # ax2.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # ax3.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # ax4.pie(city_jobs_count, labels = cities,autopct='%1.2f%%') # Save it to a temporary buffer. buf = BytesIO() fig2.savefig(buf, format="png") # Embed the result in the html output. data2 = base64.b64encode(buf.getbuffer()).decode("ascii") return render_template("charts/pie.html", pie=data, hist=data2)
async def on_member_join(mem): im = Image.open("./banner2.png") draw = ImageDraw.Draw(im, mode='RGBA') text = mem.display_name fsiz = 48 font=ImageFont.truetype("./Starfish.ttf", fsiz) while draw.textsize(text, font=font)[0] > 430: fsiz -= 1 font=ImageFont.truetype("./Starfish.ttf", fsiz) tx, ty = draw.textsize(text, font=font) x = 250 - tx//2 y = round(158 * 1.8) - ty//2 #shadowcolor = (100, 100, 100) #shadowcolor = (255,255,255) fillcolor = (165, 214, 254) shadowcolor = (105, 154, 194) a = "center" draw.text((x-1, y-1), text, font=font, fill=shadowcolor, align=a) draw.text((x+1, y+1), text, font=font, fill=shadowcolor, align=a) draw.text((x+1, y-1), text, font=font, fill=shadowcolor, align=a) draw.text((x-1, y+1), text, font=font, fill=shadowcolor, align=a) draw.text((x, y), text, font=font, fill=fillcolor, align=a) avatar_im = None url = mem.avatar_url if not url: url = mem.default_avatar_url while True: async with aiohttp.ClientSession(loop=bot.loop) as aiosession: with aiohttp.Timeout(10): async with aiosession.get(url) as resp: avatar_im = BytesIO(await resp.read()) if avatar_im.getbuffer().nbytes > 0 or retries == 0: await aiosession.close() break retries -= 1 print('0 nbytes image found. Retries left: {}'.format(retries+1)) ava_sqdim = 78 resize = (ava_sqdim, ava_sqdim) avatar_im = Image.open(avatar_im).convert("RGBA") avatar_im = avatar_im.resize(resize, Image.ANTIALIAS) avatar_im.putalpha(avatar_im.split()[3]) is_square = False if not is_square: mask = Image.new('L', resize, 0) maskDraw = ImageDraw.Draw(mask) maskDraw.ellipse((0, 0) + resize, fill=255) mask = mask.resize(avatar_im.size, Image.ANTIALIAS) avatar_im.putalpha(mask) img_center_x = (im.width // 2) img_center_y = (im.height // 2) offset_x = 109 offset_y = 36 img_offset_x = img_center_x + offset_x img_offset_y = img_center_y + offset_y ava_right = img_offset_x + avatar_im.width//2 ava_bottom = img_offset_y + avatar_im.height//2 ava_left = img_offset_x - avatar_im.width//2 ava_top = img_offset_y - avatar_im.height//2 avatar_im = tint_image(avatar_im, (255, 255, 255, 80)) im.paste(avatar_im, box=(ava_left, ava_top, ava_right, ava_bottom), mask=avatar_im) temp = BytesIO() im.save(temp, format="png") temp.seek(0) await bot.send_file(mem.server.default_channel ,temp, content="Give a popping welcome to " + mem.display_name + " :candy:", filename="welcome.png")
def plotting(): if request.method == 'GET': form = PlottingForm() return render_template('plotting.html', title='Kuvaaja', form=form) elif request.method == 'POST': form = PlottingForm() print('POST') if form.validate_on_submit(): print('form validated') # get the coefficients if form.const.data: const = form.const.data else: const = 0 if form.x1.data: x1 = form.x1.data else: x1 = 0 if form.x2.data: x2 = form.x2.data else: x2 = 0 if form.x3.data: x3 = form.x3.data else: x3 = 0 if form.x4.data: x4 = form.x4.data else: x4 = 0 if form.x_start.data: x_start = form.x_start.data else: x_start = -5 if form.x_end.data: x_end = form.x_end.data else: x_end = 5 # form the figure title otsikko = '' for coeff, nome in zip([x4, x3, x2, x1, const], ['x^4', 'x^3', 'x^2', 'x', '']): if coeff != 0: if coeff > 0: otsikko = otsikko + str(coeff) + nome + " + " else: otsikko = otsikko[:-2] + "- " + str(coeff).lstrip( '-') + nome + " + " otsikko = f"y = {otsikko.rstrip(' +*')} x:n arvoilla {x_start} ≤ x ≤ {x_end}".replace( '.', ',') print('otsikko: ', otsikko) # create the data to plot x = np.linspace(x_start, x_end, 100) y = const + x1 * x + x2 * x**2 + x3 * x**3 + x4 * x**4 # generate the figure fig = Figure() ax = fig.subplots() ax.plot(x, y) ax.set_title(otsikko) ax.axhline(y=0, color='k') ax.axvline(x=0, color='k') ax.set_xlabel('x') ax.set_ylabel('y') ax.grid() # save it to a temporary buffer buf = BytesIO() fig.savefig(buf, format='png') # prepare the data for html output data = base64.b64encode(buf.getbuffer()).decode("ascii") return render_template('plotting.html', title='Kuvaaja', form=form, data=data) flash('Tarkista syöttämäsi arvot.') return render_template('plotting.html', title='Kuvaaja', form=form)
class Headers: header_size = 112 chunk_size = 10**16 max_target = 0x0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff genesis_hash = b'9c89283ba0f3227f6c03b70216b9f665f0118d5e0fa729cedf4fb34d6a34f463' target_timespan = 150 checkpoints = HASHES first_block_timestamp = 1466646588 # block 1, as 0 is off by a lot timestamp_average_offset = 160.6855883050695 # calculated at 733447 validate_difficulty: bool = True def __init__(self, path) -> None: self.io = None self.path = path self._size: Optional[int] = None self.chunk_getter: Optional[Callable] = None self.known_missing_checkpointed_chunks = set() self.check_chunk_lock = asyncio.Lock() async def open(self): self.io = BytesIO() if self.path != ':memory:': def _readit(): if os.path.exists(self.path): with open(self.path, 'r+b') as header_file: self.io.seek(0) self.io.write(header_file.read()) await asyncio.get_event_loop().run_in_executor(None, _readit) bytes_size = self.io.seek(0, os.SEEK_END) self._size = bytes_size // self.header_size max_checkpointed_height = max(self.checkpoints.keys() or [-1]) + 1000 if bytes_size % self.header_size: log.warning("Reader file size doesnt match header size. Repairing, might take a while.") await self.repair() else: # try repairing any incomplete write on tip from previous runs (outside of checkpoints, that are ok) await self.repair(start_height=max_checkpointed_height) await self.ensure_checkpointed_size() await self.get_all_missing_headers() async def close(self): if self.io is not None: def _close(): flags = 'r+b' if os.path.exists(self.path) else 'w+b' with open(self.path, flags) as header_file: header_file.write(self.io.getbuffer()) await asyncio.get_event_loop().run_in_executor(None, _close) self.io.close() self.io = None @staticmethod def serialize(header): return b''.join([ struct.pack('<I', header['version']), unhexlify(header['prev_block_hash'])[::-1], unhexlify(header['merkle_root'])[::-1], unhexlify(header['claim_trie_root'])[::-1], struct.pack('<III', header['timestamp'], header['bits'], header['nonce']) ]) @staticmethod def deserialize(height, header): version, = struct.unpack('<I', header[:4]) timestamp, bits, nonce = struct.unpack('<III', header[100:112]) return { 'version': version, 'prev_block_hash': hexlify(header[4:36][::-1]), 'merkle_root': hexlify(header[36:68][::-1]), 'claim_trie_root': hexlify(header[68:100][::-1]), 'timestamp': timestamp, 'bits': bits, 'nonce': nonce, 'block_height': height, } def get_next_chunk_target(self, chunk: int) -> ArithUint256: return ArithUint256(self.max_target) def get_next_block_target(self, max_target: ArithUint256, previous: Optional[dict], current: Optional[dict]) -> ArithUint256: # https://github.com/lbryio/lbrycrd/blob/master/src/lbry.cpp if previous is None and current is None: return max_target if previous is None: previous = current actual_timespan = current['timestamp'] - previous['timestamp'] modulated_timespan = self.target_timespan + int((actual_timespan - self.target_timespan) / 8) minimum_timespan = self.target_timespan - int(self.target_timespan / 8) # 150 - 18 = 132 maximum_timespan = self.target_timespan + int(self.target_timespan / 2) # 150 + 75 = 225 clamped_timespan = max(minimum_timespan, min(modulated_timespan, maximum_timespan)) target = ArithUint256.from_compact(current['bits']) new_target = min(max_target, (target * clamped_timespan) / self.target_timespan) return new_target def __len__(self) -> int: return self._size def __bool__(self): return True async def get(self, height) -> dict: if isinstance(height, slice): raise NotImplementedError("Slicing of header chain has not been implemented yet.") try: return self.deserialize(height, await self.get_raw_header(height)) except struct.error: raise IndexError(f"failed to get {height}, at {len(self)}") def estimated_timestamp(self, height, try_real_headers=True): if height <= 0: return if try_real_headers and self.has_header(height): offset = height * self.header_size return struct.unpack('<I', self.io.getbuffer()[offset + 100: offset + 104])[0] return int(self.first_block_timestamp + (height * self.timestamp_average_offset)) def estimated_julian_day(self, height): return date_to_julian_day(date.fromtimestamp(self.estimated_timestamp(height, False))) async def get_raw_header(self, height) -> bytes: if self.chunk_getter: await self.ensure_chunk_at(height) if not 0 <= height <= self.height: raise IndexError(f"{height} is out of bounds, current height: {self.height}") return self._read(height) def _read(self, height, count=1): offset = height * self.header_size return bytes(self.io.getbuffer()[offset: offset + self.header_size * count]) def chunk_hash(self, start, count): return self.hash_header(self._read(start, count)).decode() async def ensure_checkpointed_size(self): max_checkpointed_height = max(self.checkpoints.keys() or [-1]) if self.height < max_checkpointed_height: self._write(max_checkpointed_height, bytes([0] * self.header_size * 1000)) async def ensure_chunk_at(self, height): async with self.check_chunk_lock: if self.has_header(height): log.debug("has header %s", height) return return await self.fetch_chunk(height) async def fetch_chunk(self, height): log.info("on-demand fetching height %s", height) start = (height // 1000) * 1000 headers = await self.chunk_getter(start) # pylint: disable=not-callable chunk = ( zlib.decompress(base64.b64decode(headers['base64']), wbits=-15, bufsize=600_000) ) chunk_hash = self.hash_header(chunk).decode() if self.checkpoints.get(start) == chunk_hash: self._write(start, chunk) if start in self.known_missing_checkpointed_chunks: self.known_missing_checkpointed_chunks.remove(start) return elif start not in self.checkpoints: return # todo: fixme raise Exception( f"Checkpoint mismatch at height {start}. Expected {self.checkpoints[start]}, but got {chunk_hash} instead." ) def has_header(self, height): normalized_height = (height // 1000) * 1000 if normalized_height in self.checkpoints: return normalized_height not in self.known_missing_checkpointed_chunks empty = '56944c5d3f98413ef45cf54545538103cc9f298e0575820ad3591376e2e0f65d' all_zeroes = '789d737d4f448e554b318c94063bbfa63e9ccda6e208f5648ca76ee68896557b' return self.chunk_hash(height, 1) not in (empty, all_zeroes) async def get_all_missing_headers(self): # Heavy operation done in one optimized shot for chunk_height, expected_hash in reversed(list(self.checkpoints.items())): if chunk_height in self.known_missing_checkpointed_chunks: continue if self.chunk_hash(chunk_height, 1000) != expected_hash: self.known_missing_checkpointed_chunks.add(chunk_height) return self.known_missing_checkpointed_chunks @property def height(self) -> int: return len(self)-1 @property def bytes_size(self): return len(self) * self.header_size async def hash(self, height=None) -> bytes: return self.hash_header( await self.get_raw_header(height if height is not None else self.height) ) @staticmethod def hash_header(header: bytes) -> bytes: if header is None: return b'0' * 64 return hexlify(double_sha256(header)[::-1]) async def connect(self, start: int, headers: bytes) -> int: added = 0 bail = False for height, chunk in self._iterate_chunks(start, headers): try: # validate_chunk() is CPU bound and reads previous chunks from file system await self.validate_chunk(height, chunk) except InvalidHeader as e: bail = True chunk = chunk[:(height-e.height)*self.header_size] if chunk: added += self._write(height, chunk) if bail: break return added def _write(self, height, verified_chunk): self.io.seek(height * self.header_size, os.SEEK_SET) written = self.io.write(verified_chunk) // self.header_size # self.io.truncate() # .seek()/.write()/.truncate() might also .flush() when needed # the goal here is mainly to ensure we're definitely flush()'ing self.io.flush() self._size = max(self._size or 0, self.io.tell() // self.header_size) return written async def validate_chunk(self, height, chunk): previous_hash, previous_header, previous_previous_header = None, None, None if height > 0: raw = await self.get_raw_header(height-1) previous_header = self.deserialize(height-1, raw) previous_hash = self.hash_header(raw) if height > 1: previous_previous_header = await self.get(height-2) chunk_target = self.get_next_chunk_target(height // 2016 - 1) for current_hash, current_header in self._iterate_headers(height, chunk): block_target = self.get_next_block_target(chunk_target, previous_previous_header, previous_header) self.validate_header(height, current_hash, current_header, previous_hash, block_target) previous_previous_header = previous_header previous_header = current_header previous_hash = current_hash def validate_header(self, height: int, current_hash: bytes, header: dict, previous_hash: bytes, target: ArithUint256): if previous_hash is None: if self.genesis_hash is not None and self.genesis_hash != current_hash: raise InvalidHeader( height, f"genesis header doesn't match: {current_hash.decode()} " f"vs expected {self.genesis_hash.decode()}") return if header['prev_block_hash'] != previous_hash: raise InvalidHeader( height, "previous hash mismatch: {} vs expected {}".format( header['prev_block_hash'].decode(), previous_hash.decode()) ) if self.validate_difficulty: if header['bits'] != target.compact: raise InvalidHeader( height, "bits mismatch: {} vs expected {}".format( header['bits'], target.compact) ) proof_of_work = self.get_proof_of_work(current_hash) if proof_of_work > target: raise InvalidHeader( height, f"insufficient proof of work: {proof_of_work.value} vs target {target.value}" ) async def repair(self, start_height=0): previous_header_hash = fail = None batch_size = 36 for height in range(start_height, self.height, batch_size): headers = self._read(height, batch_size) if len(headers) % self.header_size != 0: headers = headers[:(len(headers) // self.header_size) * self.header_size] for header_hash, header in self._iterate_headers(height, headers): height = header['block_height'] if previous_header_hash: if header['prev_block_hash'] != previous_header_hash: fail = True elif height == 0: if header_hash != self.genesis_hash: fail = True else: # for sanity and clarity, since it is the only way we can end up here assert start_height > 0 and height == start_height if fail: log.warning("Header file corrupted at height %s, truncating it.", height - 1) self.io.seek(max(0, (height - 1)) * self.header_size, os.SEEK_SET) self.io.truncate() self.io.flush() self._size = self.io.seek(0, os.SEEK_END) // self.header_size return previous_header_hash = header_hash @classmethod def get_proof_of_work(cls, header_hash: bytes): return ArithUint256(int(b'0x' + cls.header_hash_to_pow_hash(header_hash), 16)) def _iterate_chunks(self, height: int, headers: bytes) -> Iterator[Tuple[int, bytes]]: assert len(headers) % self.header_size == 0, f"{len(headers)} {len(headers)%self.header_size}" start = 0 end = (self.chunk_size - height % self.chunk_size) * self.header_size while start < end: yield height + (start // self.header_size), headers[start:end] start = end end = min(len(headers), end + self.chunk_size * self.header_size) def _iterate_headers(self, height: int, headers: bytes) -> Iterator[Tuple[bytes, dict]]: assert len(headers) % self.header_size == 0, len(headers) for idx in range(len(headers) // self.header_size): start, end = idx * self.header_size, (idx + 1) * self.header_size header = headers[start:end] yield self.hash_header(header), self.deserialize(height+idx, header) @staticmethod def header_hash_to_pow_hash(header_hash: bytes): header_hash_bytes = unhexlify(header_hash)[::-1] h = sha512(header_hash_bytes) pow_hash = double_sha256( ripemd160(h[:len(h) // 2]) + ripemd160(h[len(h) // 2:]) ) return hexlify(pow_hash[::-1])
def binary_search(self, picture, size_target, dimension, dimension_factor, rotation, i, max_i, quality, L, R, im_buffer=None): # It's possible that the picture file size is already less than the target # file size, but we can still rotate the image here. if picture.size < size_target: print("{} is already less than {} bytes".format( picture, size_target)) im = Image.open(picture) if rotation == 90: im = im.transpose(Image.ROTATE_90) elif rotation == 180: im = im.transpose(Image.ROTATE_180) elif rotation == 270: im = im.transpose(Image.ROTATE_270) im_buffer = BytesIO() im.save(im_buffer, "JPEG", quality=quality) return im_buffer # If the maximum number of iterations have been reached, return if i > max_i: print("Max iterations have been reached for {}".format(picture)) return im_buffer # Open the image file, alter its dimensions, and save it as a new BytesIO file # named 'im_buffer'. if quality <= 95: im = Image.open(picture) if rotation == 90: im = im.transpose(Image.ROTATE_90) elif rotation == 180: im = im.transpose(Image.ROTATE_180) elif rotation == 270: im = im.transpose(Image.ROTATE_270) new_dimension = (dimension[0] * dimension_factor, dimension[1] * dimension_factor) im.thumbnail(new_dimension, Image.ANTIALIAS) # new_prefix = '{}x-'.format(dimension_factor) # new_name = new_prefix + name + '-' + str(dimension[0]) + '.jpg' im_buffer = BytesIO() im.save(im_buffer, "JPEG", quality=quality) # Use L and R pointers to move closer to a value for the 'quality' parameter # that produces an image with a file size, in bytes, as close to size_target # as possible using a binary search-type of algorithm. if im_buffer.getbuffer().nbytes < size_target: print( 'Resulting image size is LESS than {} bytes:'.format( size_target), im_buffer.getbuffer().nbytes, 'bytes, quality =', quality) L = quality quality = int((R + L) / 2) return self.binary_search(picture, size_target, dimension, dimension_factor, rotation, i + 1, max_i, quality, L, R, im_buffer) elif im_buffer.getbuffer().nbytes > size_target: print( 'Resulting image size is GREATER than {} bytes:'.format( size_target), im_buffer.getbuffer().nbytes, 'bytes, quality =', quality) R = quality quality = int((R + L) / 2) return self.binary_search(picture, size_target, dimension, dimension_factor, rotation, i + 1, max_i, quality, L, R, im_buffer) else: print( 'Resulting image size EQUALS {} bytes:'.format( size_target), im_buffer.getbuffer().nbytes, 'bytes, quality =', quality) return im_buffer else: return im_buffer
def _write_to_s3(time: datetime, key: OutputGroupingKey, events: List[EngineResult]) -> None: # 'version', 'title', 'dedup_period' of a rule might differ if the rule was modified # while the rules engine was running. We pick the first encountered set of values. group_info = MatchingGroupInfo( rule_id=key.rule_id, rule_version=events[0].rule_version, log_type=key.log_type, dedup=key.dedup, dedup_period_mins=events[0].dedup_period_mins, num_matches=len(events), title=events[0].title, processing_time=time, is_rule_error=key.is_rule_error) alert_info = update_get_alert_info(group_info) data_stream = BytesIO() writer = gzip.GzipFile(fileobj=data_stream, mode='wb') for event in events: serialized_data = _serialize_event(event, alert_info) writer.write(serialized_data) writer.close() data_stream.seek(0) output_uuid = uuid.uuid4() if key.is_rule_error: key_format = _RULE_ERRORS_KEY_FORMAT else: key_format = _RULE_MATCHES_KEY_FORMAT object_key = key_format.format(key.table_name(), time.year, time.month, time.day, time.hour, key.rule_id, time.strftime(_S3_KEY_DATE_FORMAT), output_uuid) byte_size = data_stream.getbuffer().nbytes # Write data to S3 _S3_CLIENT.put_object(Bucket=_S3_BUCKET, ContentType='gzip', Body=data_stream, Key=object_key) # If the object written contains events that caused a rule engine error # don't send an SNS notification. TODO: Remove this condition and send a notification once # the backend supports rule errors end to end. if key.is_rule_error: return # Send notification to SNS topic notification = _s3_put_object_notification(_S3_BUCKET, object_key, byte_size) # MessageAttributes are required so that subscribers to SNS topic can filter events in the subscription _SNS_CLIENT.publish(TopicArn=_SNS_TOPIC_ARN, Message=json.dumps(notification), MessageAttributes={ 'type': { 'DataType': 'String', 'StringValue': 'RuleMatches' }, 'id': { 'DataType': 'String', 'StringValue': key.rule_id } })
def order_report(request): base = request.POST.get("base") quote = request.POST.get("quote") exchange = "KuCoin" # only KuCoin provides dataset of orders value_lower_bound = float(request.POST.get("value_lower_bound")) value_upper_bound = float(request.POST.get("value_upper_bound")) value_nbins = int(request.POST.get("value_nbins")) quantity_lower_bound = float(request.POST.get("quantity_lower_bound")) quantity_upper_bound = float(request.POST.get("quantity_upper_bound")) quantity_nbins = int(request.POST.get("quantity_nbins")) symbol = Symbol.objects.get(base__name__iexact=base, quote__name__iexact=quote) exchange = Exchange.objects.get(name__iexact=exchange) df = pd.DataFrame( list( Order.objects.filter(exchange=exchange, symbol=symbol).values("price", "amount", "time"))) df["price"] = pd.to_numeric(df["price"]) df["quantity"] = pd.to_numeric(df["amount"]) del df["amount"] df["value"] = df["price"] * df["quantity"] df.set_index("time", inplace=True) value_size = df.loc[(df["value"] >= value_lower_bound) & (df["value"] <= value_upper_bound)]["value"] value_bins = pd.Series( np.linspace(value_lower_bound, value_upper_bound, value_nbins + 1)) value_out = pd.cut(value_size, value_bins) value_out_norm = value_out.value_counts(sort=False) value_x = value_out_norm.index value_y = value_out_norm plt.xticks(rotation=90, fontsize=6) sns.barplot(x=value_x, y=value_y) plt.xlabel("Value Range of Orders") plt.ylabel("Count") plt.tight_layout() buf = BytesIO() plt.savefig(buf, format="png", dpi=settings.PLOT_DPI) value_data = base64.b64encode(buf.getbuffer()).decode("ascii") plt.show(block=False) quantity_size = df.loc[(df["quantity"] >= quantity_lower_bound) & ( df["quantity"] <= quantity_upper_bound)]["quantity"] quantity_bins = pd.Series( np.linspace(quantity_lower_bound, quantity_upper_bound, quantity_nbins + 1)) quantity_out = pd.cut(quantity_size, quantity_bins) quantity_out_norm = quantity_out.value_counts(sort=False) quantity_x = quantity_out_norm.index quantity_y = quantity_out_norm plt.xticks(rotation=90, fontsize=6) sns.barplot(x=quantity_x, y=quantity_y) plt.xlabel("Quantity Range of Orders") plt.ylabel("Count") plt.tight_layout() buf = BytesIO() plt.savefig(buf, format="png", dpi=settings.PLOT_DPI) quantity_data = base64.b64encode(buf.getbuffer()).decode("ascii") plt.show(block=False) return JsonResponse({ "html": render_to_string( "market/order_report.html", { "order_num": len(df), "value_src": f"data:image/png;base64,{value_data}", "quantity_src": f"data:image/png;base64,{quantity_data}", "symbol": symbol, "exchange": exchange, "describe": df.describe().to_html( classes= "table is-bordered is-striped is-hoverable is-fullwidth". split()), }) })
class DolFile(object): def __init__(self, f): self._rawdata = BytesIO(f.read()) f.seek(0) fileoffset = 0 addressoffset = 0x48 sizeoffset = 0x90 self._text = [] self._data = [] nomoretext = False nomoredata = False self._current_end = None # Read text and data section addresses and sizes for i in range(18): f.seek(fileoffset + i * 4) offset = read_uint32(f) f.seek(addressoffset + i * 4) address = read_uint32(f) f.seek(sizeoffset + i * 4) size = read_uint32(f) if i <= 6: if offset != 0: self._text.append((offset, address, size)) # print("text{0}".format(i), hex(offset), hex(address), hex(size)) else: datanum = i - 7 if offset != 0: self._data.append((offset, address, size)) # print("data{0}".format(datanum), hex(offset), hex(address), hex(size)) f.seek(0xD8) self.bssaddr = read_uint32(f) self.bsssize = read_uint32(f) #self.bss = BytesIO(self._rawdata.getbuffer()[self._bssaddr:self._bssaddr+self.bsssize]) self._curraddr = self._text[0][1] self.seek(self._curraddr) @property def sections(self): for i in self._text: yield i for i in self._data: yield i return # Internal function for resolving a gc address def _resolve_address(self, gc_addr): for offset, address, size in self.sections: if address <= gc_addr < address + size: return offset, address, size """for offset, address, size in self._text: if address <= gc_addr < address+size: return offset, address, size for offset, address, size in self._data: if address <= gc_addr < address+size: return offset, address, size """ raise UnmappedAddress("Unmapped address: {0}".format(hex(gc_addr))) def _adjust_header(self): curr = self._rawdata.tell() fileoffset = 0 addressoffset = 0x48 sizeoffset = 0x90 f = self._rawdata i = 0 for offset, address, size in self._text: f.seek(fileoffset + i * 4) write_uint32(f, offset) f.seek(addressoffset + i * 4) write_uint32(f, address) f.seek(sizeoffset + i * 4) write_uint32(f, size) i += 1 i = 7 for offset, address, size in self._data: f.seek(fileoffset + i * 4) write_uint32(f, offset) f.seek(addressoffset + i * 4) write_uint32(f, address) f.seek(sizeoffset + i * 4) write_uint32(f, size) i += 1 f.seek(0xD8) write_uint32(f, self.bssaddr) write_uint32(f, self.bsssize) f.seek(curr) # Unsupported: Reading an entire dol file # Assumption: A read should not go beyond the current section def read(self, size): if self._curraddr + size > self._current_end: raise RuntimeError("Read goes over current section") return self._rawdata.read(size) self._curraddr += size # Assumption: A write should not go beyond the current section def write(self, data): if self._curraddr + len(data) > self._current_end: raise RuntimeError("Write goes over current section") self._rawdata.write(data) self._curraddr += len(data) def seek(self, addr): offset, gc_start, gc_size = self._resolve_address(addr) self._rawdata.seek(offset + (addr - gc_start)) self._curraddr = addr self._current_end = gc_start + gc_size def _add_section(self, newsize, section, addr=None): if addr is not None: last_addr = addr else: last_addr = 0 last_offset = 0 for offset, address, size in self.sections: if last_addr < address + size: last_addr = address + size if last_offset < offset + size: last_offset = offset + size if last_addr < self.bssaddr + self.bsssize: last_addr = self.bssaddr + self.bsssize section.append((last_offset, last_addr, newsize)) curr = self._rawdata.tell() self._rawdata.seek(last_offset) self._rawdata.write(b" " * newsize) self._rawdata.seek(curr) return (last_offset, last_addr, newsize) def allocate_text_section(self, size, addr=None): assert len(self._text) <= 7 if len(self._text) >= 7: raise SectionCountFull("Maximum amount of text sections reached!") return self._add_section(size, self._text, addr) def allocate_data_section(self, size, addr=None): assert len(self._data) <= 11 if len(self._data) >= 11: raise SectionCountFull("Maximum amount of data sections reached!") return self._add_section(size, self._data, addr=None) def tell(self): return self._curraddr def save(self, f): self._adjust_header() f.write(self._rawdata.getbuffer()) def print_info(self): print("Dol Info:") i = 0 for offset, addr, size in self._text: print("text{0}: fileoffset {1:x}, addr {2:x}, size {3:x}".format( i, offset, addr, size)) i += 1 i = 0 for offset, addr, size in self._data: print("data{0}: fileoffset {1:x}, addr {2:x}, size {3:x}".format( i, offset, addr, size)) i += 1 print("bss addr: {0:x}, bss size: {1:x}, bss end: {2:x}".format( self.bssaddr, self.bsssize, self.bssaddr + self.bsssize))
def create_graph(name, name1, stat, stat1, id): fig = Figure() ax = fig.subplots() bar1 = get_data(name, stat) session_time_name = get_data(name, 'datetime') session_time_name_1 = get_data(name1, 'datetime') session_time = session_time_name if len(session_time_name) != len(session_time_name_1): if len(session_time_name) >= len(session_time_name_1): session_time = session_time_name if len(session_time_name_1) >= len(session_time_name): session_time = session_time_name_1 labels = [] bar1_list = [] bar2_list = [] if id == 2: bar2 = get_data(name1, stat1) if len(bar1) >= len(bar2): difference = int(len(bar1) - len(bar2)) if len(bar2) >= len(bar1): difference = int(len(bar2) - len(bar1)) if len(bar1) != len(bar2): for x in range(difference): if len(bar1) <= len(bar2): bar1_list.append(int(0)) elif len(bar1) >= len(bar2): bar2_list.append(int(0)) elif len(bar2) <= len(bar1): bar2_list.append(int(0)) elif len(bar2) >= len(bar1): bar1_list.append(0) for st2 in bar2: for s2 in st2: bar2_list.append(s2) for st1 in bar1: for s1 in st1: bar1_list.append(s1) for label in session_time: for l in label: labels.append(l) x = np.arange(len(bar1_list)) width = 0.35 if id == 1: rects1 = ax.bar(x - width / 800, bar1_list, width, label=f"{name} {stat}") if id == 2: rects1 = ax.bar(x - width / 2, bar1_list, width, label=f"{name} {stat}") rects2 = ax.bar(x + width / 2, bar2_list, width, label=f"{name1} {stat1}") ax.set_ylabel('Stats') ax.set_xticks(x) ax.set_xticklabels(labels) ax.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc='lower left', ncol=2, mode="expand", borderaxespad=0.) ax.bar_label(rects1, padding=3) if id == 2: ax.bar_label(rects2, padding=3) buf = BytesIO() fig.savefig(buf, format="png") data = base64.b64encode(buf.getbuffer()).decode("ascii") labels.clear() bar1_list.clear() bar2_list.clear() return html + f"""
class AnimFile: def __init__(self, split=False, old=True): self.old = old self.split = split if old: self.raw_data = BytesIO() else: self.afm2 = AFM2() self.afsa, self.afsb = (AFSA(), AFSB()) if split else (None, None) def read(self, f): if self.old: self.raw_data.write(f.read()) self.raw_data.seek(0) else: while True: try: magic = f.read(4).decode('utf-8') except EOFError: break except struct.error: break except UnicodeDecodeError: print('\nAttempted reading non-chunked data.') break if not magic: break magic_lower = magic.lower() local_chunk = getattr(self, magic_lower) if not local_chunk: setattr(self, magic_lower, chunk().read(f)) else: local_chunk.read(f) return self def write(self, f): if self.old: f.write(bytes(self.raw_data.getbuffer())) else: self.afm2.write(f) if self.afsa: self.afsa.write(f) if self.afsb: self.afsb.write(f) return self
w = BytesIO() w.write(magic) # write the TLV for file in files: filepath = f"{dirpath}{os.sep}{file}" with open(filepath, 'rb') as f: file = os.path.basename(file) b = f.read() z = zlib.compress(b) w.write( struct.pack(f'<{len(file) + 1}sII{len(z)}s', file.encode('utf8') + b'\x00', len(z), len(b), z)) charset = (string.ascii_letters + string.digits).encode() data_buf = w.getbuffer()[len(magic):] # otp = [random.choice(charset) for i in range(len(data_buf))] otp = [] with open('./baby-parser3/otp.key', 'rb') as f: otp = list(f.read()) buf = [b ^ otp[i] for i, b in enumerate(data_buf)] w.getbuffer()[len(magic):] = bytes(buf) # with open('./build/otp.key', 'wb') as f: # f.write(bytes(otp)) os.makedirs('build', exist_ok=True) with open('./build/baby-parser3.wh', 'wb') as f: f.write(w.getvalue())
def do_GET(self): lastlimit = "00" lastdetect = "00" downscale = argdownscale matches = 2 possiblematch = "00" timeInterval = 1 lastTime = 0 if self.path.endswith('.mjpg'): self.send_response(200) self.send_header( 'Content-type', 'multipart/x-mixed-replace; boundary=--jpgboundary') self.end_headers() while True: try: #tästä poistettu koodi muistiossa rc, img = capture.read() origframe = img img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (128, 128)) comp = "Unknown" comp, amnt = run_flann(img) if comp != "Unknown": if comp != lastlimit: if comp == lastdetect: possiblematch = comp matches = matches + 1 if matches >= argmatches: lastlimit = possiblematch matches = 0 else: possiblematch = "00" matches = 0 else: #print("Unknow speed limit") comp = lastdetect lastdetect = comp cv2.putText( origframe, "Current speed limit: " + str(lastlimit) + " km/h.", (5, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2) imgRGB = origframe imgRGB = cv2.resize(imgRGB, (320, 240)) jpg = Image.fromarray(imgRGB) tmpFile = BytesIO() jpg.save(tmpFile, 'JPEG') self.wfile.write("--jpgboundary".encode()) self.send_header('Content-type', 'image/jpeg') self.send_header('Content-length', str(tmpFile.getbuffer().nbytes)) self.end_headers() jpg.save(self.wfile, 'JPEG') except KeyboardInterrupt: break return if self.path.endswith('.html'): self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write('<html><head></head><body>'.encode()) self.wfile.write( '<img src="http://172.20.10.9:8080/cam.mjpg"/>'.encode()) self.wfile.write('</body></html>'.encode()) return
def update_redis_stats(): """Generate the system stats and save them in redis Returns ------- list of str artifact filepaths that are not present in the file system """ STUDY = qdb.study.Study number_studies = {'public': 0, 'private': 0, 'sandbox': 0} number_of_samples = {'public': 0, 'private': 0, 'sandbox': 0} num_studies_ebi = 0 num_samples_ebi = 0 number_samples_ebi_prep = 0 stats = [] missing_files = [] per_data_type_stats = Counter() for study in STUDY.iter(): st = study.sample_template if st is None: continue # counting samples submitted to EBI-ENA len_samples_ebi = sum( [esa is not None for esa in st.ebi_sample_accessions.values()]) if len_samples_ebi != 0: num_studies_ebi += 1 num_samples_ebi += len_samples_ebi samples_status = defaultdict(set) for pt in study.prep_templates(): pt_samples = list(pt.keys()) pt_status = pt.status if pt_status == 'public': per_data_type_stats[pt.data_type()] += len(pt_samples) samples_status[pt_status].update(pt_samples) # counting experiments (samples in preps) submitted to EBI-ENA number_samples_ebi_prep += sum([ esa is not None for esa in pt.ebi_experiment_accessions.values() ]) # counting studies if 'public' in samples_status: number_studies['public'] += 1 elif 'private' in samples_status: number_studies['private'] += 1 else: # note that this is a catch all for other status; at time of # writing there is status: awaiting_approval number_studies['sandbox'] += 1 # counting samples; note that some of these lines could be merged with # the block above but I decided to split it in 2 for clarity if 'public' in samples_status: number_of_samples['public'] += len(samples_status['public']) if 'private' in samples_status: number_of_samples['private'] += len(samples_status['private']) if 'sandbox' in samples_status: number_of_samples['sandbox'] += len(samples_status['sandbox']) # processing filepaths for artifact in study.artifacts(): for adata in artifact.filepaths: try: s = stat(adata['fp']) except OSError: missing_files.append(adata['fp']) else: stats.append((adata['fp_type'], s.st_size, strftime('%Y-%m', localtime(s.st_mtime)))) num_users = qdb.util.get_count('qiita.qiita_user') num_processing_jobs = qdb.util.get_count('qiita.processing_job') lat_longs = dumps(get_lat_longs()) summary = {} all_dates = [] # these are some filetypes that are too small to plot alone so we'll merge # in other group_other = { 'html_summary', 'tgz', 'directory', 'raw_fasta', 'log', 'biom', 'raw_sff', 'raw_qual', 'qza', 'html_summary_dir', 'qza', 'plain_text', 'raw_barcodes' } for ft, size, ym in stats: if ft in group_other: ft = 'other' if ft not in summary: summary[ft] = {} if ym not in summary[ft]: summary[ft][ym] = 0 all_dates.append(ym) summary[ft][ym] += size all_dates = sorted(set(all_dates)) # sorting summaries ordered_summary = {} for dt in summary: new_list = [] current_value = 0 for ad in all_dates: if ad in summary[dt]: current_value += summary[dt][ad] new_list.append(current_value) ordered_summary[dt] = new_list plot_order = sorted([(k, ordered_summary[k][-1]) for k in ordered_summary], key=lambda x: x[1]) # helper function to generate y axis, modified from: # http://stackoverflow.com/a/1094933 def sizeof_fmt(value, position): number = None for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(value) < 1024.0: number = "%3.1f%s" % (value, unit) break value /= 1024.0 if number is None: number = "%.1f%s" % (value, 'Yi') return number all_dates_axis = range(len(all_dates)) plt.locator_params(axis='y', nbins=10) plt.figure(figsize=(20, 10)) for k, v in plot_order: plt.plot(all_dates_axis, ordered_summary[k], linewidth=2, label=k) plt.xticks(all_dates_axis, all_dates) plt.legend() plt.grid() ax = plt.gca() ax.yaxis.set_major_formatter(mpl.ticker.FuncFormatter(sizeof_fmt)) plt.xticks(rotation=90) plt.xlabel('Date') plt.ylabel('Storage space per data type') plot = BytesIO() plt.savefig(plot, format='png') plot.seek(0) img = 'data:image/png;base64,' + quote(b64encode(plot.getbuffer())) time = datetime.now().strftime('%m-%d-%y %H:%M:%S') portal = qiita_config.portal # making sure per_data_type_stats has some data so hmset doesn't fail if per_data_type_stats == {}: per_data_type_stats['No data'] = 0 vals = [('number_studies', number_studies, r_client.hmset), ('number_of_samples', number_of_samples, r_client.hmset), ('per_data_type_stats', dict(per_data_type_stats), r_client.hmset), ('num_users', num_users, r_client.set), ('lat_longs', (lat_longs), r_client.set), ('num_studies_ebi', num_studies_ebi, r_client.set), ('num_samples_ebi', num_samples_ebi, r_client.set), ('number_samples_ebi_prep', number_samples_ebi_prep, r_client.set), ('img', img, r_client.set), ('time', time, r_client.set), ('num_processing_jobs', num_processing_jobs, r_client.set)] for k, v, f in vals: redis_key = '%s:stats:%s' % (portal, k) # important to "flush" variables to avoid errors r_client.delete(redis_key) f(redis_key, v) # preparing vals to insert into DB vals = dumps(dict([x[:-1] for x in vals])) with qdb.sql_connection.TRN: sql = """INSERT INTO qiita.stats_daily (stats, stats_timestamp) VALUES (%s, NOW())""" qdb.sql_connection.TRN.add(sql, [vals]) qdb.sql_connection.TRN.execute() return missing_files
def mock_text_box(mock_text_box_stream, tmpdir_factory): file = BytesIO(mock_text_box_stream) fn = tmpdir_factory.mktemp("data").join("mock_text_box_file.png") with open(fn, 'wb') as f: f.write(file.getbuffer()) return str(fn)
def mock_cord_dataset(tmpdir_factory, mock_image_stream): root = tmpdir_factory.mktemp('datasets') cord_root = root.mkdir('cord_train') image_folder = cord_root.mkdir("image") annotations_folder = cord_root.mkdir("json") labels = { "dontcare": [], "valid_line": [{ "words": [{ "quad": { "x2": 270, "y3": 390, "x3": 270, "y4": 390, "x1": 256, "y1": 374, "x4": 256, "y2": 374 }, "is_key": 0, "row_id": 2179893, "text": "I" }], "category": "menu.cnt", "group_id": 3 }, { "words": [{ "quad": { "x2": 270, "y3": 418, "x3": 270, "y4": 418, "x1": 258, "y1": 402, "x4": 258, "y2": 402 }, "is_key": 0, "row_id": 2179894, "text": "am" }], "category": "menu.cnt", "group_id": 4 }, { "words": [{ "quad": { "x2": 272, "y3": 444, "x3": 272, "y4": 444, "x1": 258, "y1": 428, "x4": 258, "y2": 428 }, "is_key": 0, "row_id": 2179895, "text": "Luke" }], "category": "menu.cnt", "group_id": 5 }] } file = BytesIO(mock_image_stream) for i in range(3): fn_i = image_folder.join(f"receipt_{i}.png") with open(fn_i, 'wb') as f: f.write(file.getbuffer()) fn_l = annotations_folder.join(f"receipt_{i}.json") with open(fn_l, 'w') as f: json.dump(labels, f) # Packing data into an archive to simulate the real data set and bypass archive extraction archive_path = root.join('cord_train.zip') shutil.make_archive(root.join('cord_train'), 'zip', str(cord_root)) return str(archive_path)