def testPacking(self): assert Msgpack.pack( self.test_data ) == b'\x85\xa3bin\xc4\np\x81zDhL\xf0O\xd0\xaf\xa3cmd\xa7fileGet\xa4list\x92\xc4\np\x81zDhL\xf0O\xd0\xaf\xc4\np\x81zDhL\xf0O\xd0\xaf\xa6params\x81\xa4site\xa51Site\xa4utf8\xad\xc3\xa1rv\xc3\xadzt\xc5\xb1r\xc5\x91' assert Msgpack.pack( self.test_data, use_bin_type=False ) == b'\x85\xa3bin\xaap\x81zDhL\xf0O\xd0\xaf\xa3cmd\xa7fileGet\xa4list\x92\xaap\x81zDhL\xf0O\xd0\xaf\xaap\x81zDhL\xf0O\xd0\xaf\xa6params\x81\xa4site\xa51Site\xa4utf8\xad\xc3\xa1rv\xc3\xadzt\xc5\xb1r\xc5\x91'
def start(self): # Listens for discover requests self.sock = self.createBroadcastSocket() if not self.sock: self.log.error("Unable to listen on port %s" % self.listen_port) return self.log.debug("Started on port %s" % self.listen_port) self.running = True while self.running: try: data, addr = self.sock.recvfrom(8192) except Exception as err: if self.running: self.log.error("Listener receive error: %s" % err) continue if not self.running: break try: message = Msgpack.unpack(data) response_addr, message = self.handleMessage(addr, message) if message: self.send(response_addr, message) except Exception as err: self.log.error("Handlemessage error: %s" % Debug.formatException(err)) self.log.debug("Stopped listening on port %s" % self.listen_port)
def send(self, message, streaming=False): self.last_send_time = time.time() if config.debug_socket: self.log( "Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) if not self.sock: self.log("Send error: missing socket") return False if not self.connected and message.get("cmd") != "handshake": self.log("Wait for handshake before send request") self.event_connected.get() try: stat_key = message.get("cmd", "unknown") if stat_key == "response": stat_key = "response: %s" % self.last_cmd_recv else: self.server.num_sent += 1 self.server.stat_sent[stat_key]["num"] += 1 if streaming: with self.send_lock: bytes_sent = Msgpack.stream(message, self.sock.sendall) self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent self.server.stat_sent[stat_key]["bytes"] += bytes_sent message = None else: data = Msgpack.pack(message) self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.server.stat_sent[stat_key]["bytes"] += len(data) message = None with self.send_lock: self.sock.sendall(data) except Exception as err: self.close("Send error: %s (cmd: %s)" % (err, stat_key)) return False self.last_sent_time = time.time() return True
def getPiecemap(self, inner_path): file_info = self.site.content_manager.getFileInfo(inner_path) piecemap_inner_path = helper.getDirname( file_info["content_inner_path"]) + file_info["piecemap"] self.site.needFile(piecemap_inner_path, priority=20) piecemap = Msgpack.unpack( self.site.storage.open( piecemap_inner_path, "rb").read())[helper.getFilename(inner_path)] piecemap["piece_size"] = file_info["piece_size"] return piecemap
def send(self, addr, message): if type(message) is not list: message = [message] for message_part in message: message_part["sender"] = self.sender_info self.log.debug("Send to %s: %s" % (addr, message_part["cmd"])) with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.sendto(Msgpack.pack(message_part), addr)
def testBackwardCompatibility(self): packed = {} packed["py3"] = Msgpack.pack(self.test_data, use_bin_type=False) packed["py3_bin"] = Msgpack.pack(self.test_data, use_bin_type=True) for key, val in packed.items(): unpacked = Msgpack.unpack(val) type(unpacked["utf8"]) == str type(unpacked["bin"]) == bytes # Packed with use_bin_type=False (pre-ZeroNet 0.7.0) unpacked = Msgpack.unpack(packed["py3"], decode=True) type(unpacked["utf8"]) == str type(unpacked["bin"]) == bytes assert len(unpacked["utf8"]) == 9 assert len(unpacked["bin"]) == 10 with pytest.raises( UnicodeDecodeError) as err: # Try to decode binary as utf-8 unpacked = Msgpack.unpack(packed["py3"], decode=False) # Packed with use_bin_type=True unpacked = Msgpack.unpack(packed["py3_bin"], decode=False) type(unpacked["utf8"]) == str type(unpacked["bin"]) == bytes assert len(unpacked["utf8"]) == 9 assert len(unpacked["bin"]) == 10
def testStreaming(self): bin_data = os.urandom(20) f = Msgpack.FilePart("%s/users.json" % config.data_dir, "rb") f.read_bytes = 30 data = {"cmd": "response", "body": f, "bin": bin_data} out_buff = io.BytesIO() Msgpack.stream(data, out_buff.write) out_buff.seek(0) data_packb = { "cmd": "response", "body": open("%s/users.json" % config.data_dir, "rb").read(30), "bin": bin_data } out_buff.seek(0) data_unpacked = Msgpack.unpack(out_buff.read()) assert data_unpacked == data_packb assert data_unpacked["cmd"] == "response" assert type(data_unpacked["body"]) == bytes
def testUnpackMsgpack(self, num_run=1): """ Test msgpack decoding """ yield "x 5KB " binary = b'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' data = OrderedDict( sorted({"int": 1024 * 1024 * 1024, "float": 12345.67890, "text": "hello" * 1024, "binary": binary}.items()) ) data_packed = b'\x84\xa6binary\xc5\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00' data_packed += b'hello' * 1024 for y in range(num_run): data_unpacked = Msgpack.unpack(data_packed, decode=False) yield "." assert data_unpacked == data, "%s<br>!=<br>%s" % (data_unpacked, data)
def testPiecemapCreate(self, site): inner_path = self.createBigfile(site) content = site.storage.loadJson("content.json") assert "data/optional.any.iso" in content["files_optional"] file_node = content["files_optional"][inner_path] assert file_node["size"] == 10 * 1000 * 1000 assert file_node[ "sha512"] == "47a72cde3be80b4a829e7674f72b7c6878cf6a70b0c58c6aa6c17d7e9948daf6" assert file_node["piecemap"] == inner_path + ".piecemap.msgpack" piecemap = Msgpack.unpack( site.storage.open(file_node["piecemap"], "rb").read())["optional.any.iso"] assert len(piecemap["sha512_pieces"]) == 10 assert piecemap["sha512_pieces"][0] != piecemap["sha512_pieces"][1] assert binascii.hexlify( piecemap["sha512_pieces"][0] ) == b"a73abad9992b3d0b672d0c2a292046695d31bebdcb1e150c8410bbe7c972eff3"
def broadcast(self, message, port=None): if not port: port = self.listen_port my_ips = self.getMyIps() addr = ("255.255.255.255", port) message["sender"] = self.sender_info self.log.debug("Broadcast using ips %s on port %s: %s" % (my_ips, port, message["cmd"])) for my_ip in my_ips: try: with closing(socket.socket(socket.AF_INET, socket.SOCK_DGRAM)) as sock: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) sock.bind((my_ip, 0)) sock.sendto(Msgpack.pack(message), addr) except Exception as err: self.log.warning("Error sending broadcast using ip %s: %s" % (my_ip, err))
def actionBigfileUpload(self): nonce = self.get.get("upload_nonce") if nonce not in upload_nonces: return self.error403("Upload nonce error.") upload_info = upload_nonces[nonce] del upload_nonces[nonce] self.sendHeader(200, "text/html", noscript=True, extra_headers={ "Access-Control-Allow-Origin": "null", "Access-Control-Allow-Credentials": "true" }) self.readMultipartHeaders(self.env['wsgi.input']) # Skip http headers site = upload_info["site"] inner_path = upload_info["inner_path"] with site.storage.open(inner_path, "wb", create_dirs=True) as out_file: merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile( self.env['wsgi.input'], upload_info["size"], upload_info["piece_size"], out_file) if len(piecemap_info["sha512_pieces"]) == 1: # Small file, don't split hash = binascii.hexlify(piecemap_info["sha512_pieces"][0]) hash_id = site.content_manager.hashfield.getHashId(hash) site.content_manager.optionalDownloaded(inner_path, hash_id, upload_info["size"], own=True) else: # Big file file_name = helper.getFilename(inner_path) site.storage.open(upload_info["piecemap"], "wb").write( Msgpack.pack({file_name: piecemap_info})) # Find piecemap and file relative path to content.json file_info = site.content_manager.getFileInfo(inner_path, new_file=True) content_inner_path_dir = helper.getDirname( file_info["content_inner_path"]) piecemap_relative_path = upload_info["piecemap"][ len(content_inner_path_dir):] file_relative_path = inner_path[len(content_inner_path_dir):] # Add file to content.json if site.storage.isFile(file_info["content_inner_path"]): content = site.storage.loadJson( file_info["content_inner_path"]) else: content = {} if "files_optional" not in content: content["files_optional"] = {} content["files_optional"][file_relative_path] = { "sha512": merkle_root, "size": upload_info["size"], "piecemap": piecemap_relative_path, "piece_size": piece_size } merkle_root_hash_id = site.content_manager.hashfield.getHashId( merkle_root) site.content_manager.optionalDownloaded(inner_path, merkle_root_hash_id, upload_info["size"], own=True) site.storage.writeJson(file_info["content_inner_path"], content) site.content_manager.contents.loadItem( file_info["content_inner_path"]) # reload cache return json.dumps({ "merkle_root": merkle_root, "piece_num": len(piecemap_info["sha512_pieces"]), "piece_size": piece_size, "inner_path": inner_path })
def getMsgpackUnpacker(self): if self.handshake and self.handshake.get("use_bin_type"): return Msgpack.getUnpacker(fallback=True, decode=False) else: # Backward compatibility for <0.7.0 return Msgpack.getUnpacker(fallback=True, decode=True)
def testUnpackinkg(self): assert Msgpack.unpack(Msgpack.pack(self.test_data)) == self.test_data
def actionBenchmark(self): import sys import gc from contextlib import contextmanager output = self.sendHeader() if "Multiuser" in PluginManager.plugin_manager.plugin_names and not config.multiuser_local: yield "This function is disabled on this proxy" return @contextmanager def benchmark(name, standard): self.log.debug("Benchmark: %s" % name) s = time.time() output(b"- %s" % name.encode()) try: yield 1 except Exception as err: self.log.exception(err) output(b"<br><b>! Error: %s</b><br>" % Debug.formatException(err).encode()) taken = time.time() - s if taken > 0: multipler = standard / taken else: multipler = 99 if multipler < 0.3: speed = "Sloooow" elif multipler < 0.5: speed = "Ehh" elif multipler < 0.8: speed = "Goodish" elif multipler < 1.2: speed = "OK" elif multipler < 1.7: speed = "Fine" elif multipler < 2.5: speed = "Fast" elif multipler < 3.5: speed = "WOW" else: speed = "Insane!!" output(b"%.3fs [x%.2f: %s]<br>" % (taken, multipler, speed.encode())) time.sleep(0.01) yield """ <style> * { font-family: monospace } table * { text-align: right; padding: 0px 10px } </style> """ yield "Benchmarking ZeroNet %s (rev%s) Python %s on: %s...<br>" % (config.version, config.rev, sys.version, sys.platform) t = time.time() # CryptBitcoin yield "<br>CryptBitcoin:<br>" from Crypt import CryptBitcoin # seed = CryptBitcoin.newSeed() # yield "- Seed: %s<br>" % seed seed = "e180efa477c63b0f2757eac7b1cce781877177fe0966be62754ffd4c8592ce38" with benchmark("hdPrivatekey x 10", 0.7): for i in range(10): privatekey = CryptBitcoin.hdPrivatekey(seed, i * 10) yield "." valid = "5JsunC55XGVqFQj5kPGK4MWgTL26jKbnPhjnmchSNPo75XXCwtk" assert privatekey == valid, "%s != %s" % (privatekey, valid) data = "Hello" * 1024 # 5k with benchmark("sign x 10", 0.35): for i in range(10): yield "." sign = CryptBitcoin.sign(data, privatekey) valid = "G1GXaDauZ8vX/N9Jn+MRiGm9h+I94zUhDnNYFaqMGuOiBHB+kp4cRPZOL7l1yqK5BHa6J+W97bMjvTXtxzljp6w=" assert sign == valid, "%s != %s" % (sign, valid) address = CryptBitcoin.privatekeyToAddress(privatekey) for lib_verify in ["btctools", "openssl", "libsecp256k1"]: try: CryptBitcoin.loadLib(lib_verify) loaded = True if lib_verify == "openssl": yield "+ Loaded lib: %s<br>" % html.escape(str(CryptBitcoin.bitcoin.core.key._ssl)) elif lib_verify == "libsecp256k1": import coincurve yield "+ Loaded lib: %s<br>" % type(coincurve._libsecp256k1.lib).__name__ except Exception as err: yield "- Error loading %s: %s<br>" % (lib_verify, err) loaded = False if not loaded: continue with benchmark("%s verify x 100" % lib_verify, 0.37): for i in range(100): if i % 10 == 0: yield "." ok = CryptBitcoin.verify(data, address, sign, lib_verify=lib_verify) assert ok, "does not verify from %s" % address # CryptHash yield "<br>CryptHash:<br>" from Crypt import CryptHash import io data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m with benchmark("sha256 5M x 10", 0.6): for i in range(10): data.seek(0) hash = CryptHash.sha256sum(data) yield "." valid = "8cd629d9d6aff6590da8b80782a5046d2673d5917b99d5603c3dcb4005c45ffa" assert hash == valid, "%s != %s" % (hash, valid) data = io.BytesIO(b"Hello" * 1024 * 1024) # 5m with benchmark("sha512 5M x 10", 0.6): for i in range(10): data.seek(0) hash = CryptHash.sha512sum(data) yield "." valid = "9ca7e855d430964d5b55b114e95c6bbb114a6d478f6485df93044d87b108904d" assert hash == valid, "%s != %s" % (hash, valid) with benchmark("os.urandom(256) x 1000", 0.0065): for i in range(10): for y in range(100): data = os.urandom(256) yield "." # Msgpack from util import Msgpack yield "<br>Msgpack: (version: %s)<br>" % ".".join(map(str, Msgpack.msgpack.version)) binary = b'fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv' data = OrderedDict( sorted({"int": 1024 * 1024 * 1024, "float": 12345.67890, "text": "hello" * 1024, "binary": binary}.items()) ) data_packed_valid = b'\x84\xa6binary\xc5\x01\x00fqv\xf0\x1a"e\x10,\xbe\x9cT\x9e(\xa5]u\x072C\x8c\x15\xa2\xa8\x93Sw)\x19\x02\xdd\t\xfb\xf67\x88\xd9\xee\x86\xa1\xe4\xb6,\xc6\x14\xbb\xd7$z\x1d\xb2\xda\x85\xf5\xa0\x97^\x01*\xaf\xd3\xb0!\xb7\x9d\xea\x89\xbbh8\xa1"\xa7]e(@\xa2\xa5g\xb7[\xae\x8eE\xc2\x9fL\xb6s\x19\x19\r\xc8\x04S\xd0N\xe4]?/\x01\xea\xf6\xec\xd1\xb3\xc2\x91\x86\xd7\xf4K\xdf\xc2lV\xf4\xe8\x80\xfc\x8ep\xbb\x82\xb3\x86\x98F\x1c\xecS\xc8\x15\xcf\xdc\xf1\xed\xfc\xd8\x18r\xf9\x80\x0f\xfa\x8cO\x97(\x0b]\xf1\xdd\r\xe7\xbf\xed\x06\xbd\x1b?\xc5\xa0\xd7a\x82\xf3\xa8\xe6@\xf3\ri\xa1\xb10\xf6\xd4W\xbc\x86\x1a\xbb\xfd\x94!bS\xdb\xaeM\x92\x00#\x0b\xf7\xad\xe9\xc2\x8e\x86\xbfi![%\xd31]\xc6\xfc2\xc9\xda\xc6v\x82P\xcc\xa9\xea\xb9\xff\xf6\xc8\x17iD\xcf\xf3\xeeI\x04\xe9\xa1\x19\xbb\x01\x92\xf5nn4K\xf8\xbb\xc6\x17e>\xa7 \xbbv\xa5float\xcb@\xc8\x1c\xd6\xe61\xf8\xa1\xa3int\xce@\x00\x00\x00\xa4text\xda\x14\x00hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohello' with benchmark("pack 5K x 10 000", 0.78): for i in range(10): for y in range(1000): data_packed = Msgpack.pack(data) yield "." assert data_packed == data_packed_valid, "%s<br>!=<br>%s" % (repr(data_packed), repr(data_packed_valid)) with benchmark("unpack 5K x 10 000", 1.2): for i in range(10): for y in range(1000): data_unpacked = Msgpack.unpack(data_packed, decode=False) yield "." assert data == data_unpacked, "%s != %s" % (data_unpacked, data) for fallback in [True, False]: with benchmark("streaming unpack 5K x 10 000 (fallback: %s)" % fallback, 1.4): for i in range(10): unpacker = Msgpack.getUnpacker(decode=False, fallback=fallback) for y in range(1000): unpacker.feed(data_packed) for data_unpacked in unpacker: pass yield "." assert data == data_unpacked, "%s != %s" % (data_unpacked, data) # Db import sqlite3 yield "<br>Db: (version: %s, API: %s)<br>" % (sqlite3.sqlite_version, sqlite3.version) schema = { "db_name": "TestDb", "db_file": "%s/benchmark.db" % config.data_dir, "maps": { ".*": { "to_table": { "test": "test" } } }, "tables": { "test": { "cols": [ ["test_id", "INTEGER"], ["title", "TEXT"], ["json_id", "INTEGER REFERENCES json (json_id)"] ], "indexes": ["CREATE UNIQUE INDEX test_key ON test(test_id, json_id)"], "schema_changed": 1426195822 } } } if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) with benchmark("Open x 10", 0.13): for i in range(10): db = Db.Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() db.close() yield "." db = Db.Db(schema, "%s/benchmark.db" % config.data_dir) db.checkTables() import json with benchmark("Insert x 10 x 1000", 1.0): for u in range(10): # 10 user data = {"test": []} for i in range(1000): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.updateJson("%s/test_%s.json" % (config.data_dir, u)) os.unlink("%s/test_%s.json" % (config.data_dir, u)) yield "." with benchmark("Buffered insert x 100 x 100", 1.3): cur = db.getCursor() cur.logging = False for u in range(100, 200): # 100 user data = {"test": []} for i in range(100): # 1000 line of data data["test"].append({"test_id": i, "title": "Testdata for %s message %s" % (u, i)}) json.dump(data, open("%s/test_%s.json" % (config.data_dir, u), "w")) db.updateJson("%s/test_%s.json" % (config.data_dir, u), cur=cur) os.unlink("%s/test_%s.json" % (config.data_dir, u)) if u % 10 == 0: yield "." yield " + Total rows in db: %s<br>" % db.execute("SELECT COUNT(*) AS num FROM test").fetchone()[0] with benchmark("Indexed query x 1000", 0.25): found = 0 cur = db.getCursor() cur.logging = False for i in range(1000): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE test_id = %s" % i) for row in res: found += 1 if i % 100 == 0: yield "." assert found == 20000, "Found: %s != 20000" % found with benchmark("Not indexed query x 100", 0.6): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE json_id = %s" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 18900, "Found: %s != 18900" % found with benchmark("Like query x 100", 1.8): found = 0 cur = db.getCursor() cur.logging = False for i in range(100): # 1000x by test_id res = cur.execute("SELECT * FROM test WHERE title LIKE '%%message %s%%'" % i) for row in res: found += 1 if i % 10 == 0: yield "." assert found == 38900, "Found: %s != 11000" % found db.close() if os.path.isfile("%s/benchmark.db" % config.data_dir): os.unlink("%s/benchmark.db" % config.data_dir) gc.collect() # Implicit grabage collection # Zip yield "<br>Compression:<br>" import zipfile test_data = b"Test" * 1024 file_name = b"\xc3\x81rv\xc3\xadzt\xc5\xb0r\xc5\x91t\xc3\xbck\xc3\xb6r\xc3\xb3g\xc3\xa9p\xe4\xb8\xad\xe5\x8d\x8e%s.txt".decode("utf8") with benchmark("Zip pack x 10", 0.12): for i in range(10): with zipfile.ZipFile('%s/test.zip' % config.data_dir, 'w') as archive: for y in range(100): zip_info = zipfile.ZipInfo(file_name % y, (1980,1,1,0,0,0)) zip_info.compress_type = zipfile.ZIP_DEFLATED zip_info.create_system = 3 zip_info.flag_bits = 0 zip_info.external_attr = 25165824 archive.writestr(zip_info, test_data) yield "." hash = CryptHash.sha512sum(open("%s/test.zip" % config.data_dir, "rb")) valid = "f630fece29fff1cc8dbf454e47a87fea2746a4dbbd2ceec098afebab45301562" assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid) with benchmark("Zip unpack x 10", 0.2): for i in range(10): with zipfile.ZipFile('%s/test.zip' % config.data_dir) as archive: for y in range(100): data = archive.open(file_name % y).read() assert archive.open(file_name % y).read() == test_data, "Invalid data: %s..." % data[0:30] yield "." if os.path.isfile("%s/test.zip" % config.data_dir): os.unlink("%s/test.zip" % config.data_dir) # gz, bz2, xz import tarfile import gzip # Monkey patch _init_write_gz to use fixed date in order to keep the hash independent from datetime def nodate_write_gzip_header(self): self._write_mtime = 0 original_write_gzip_header(self) original_write_gzip_header = gzip.GzipFile._write_gzip_header gzip.GzipFile._write_gzip_header = nodate_write_gzip_header test_data_io = io.BytesIO(b"Test" * 1024) archive_formats = { "gz": {"hash": "4704ebd8c987ed6f833059f1de9c475d443b0539b8d4c4cb8b49b26f7bbf2d19", "time_pack": 0.3, "time_unpack": 0.2}, "bz2": {"hash": "90cba0b4d9abaa37b830bf37e4adba93bfd183e095b489ebee62aaa94339f3b5", "time_pack": 2.0, "time_unpack": 0.5}, "xz": {"hash": "37abc16d552cfd4a495cb2acbf8b1d5877631d084f6571f4d6544bc548c69bae", "time_pack": 1.4, "time_unpack": 0.2} } for ext, format_data in archive_formats.items(): archive_path = '%s/test.tar.%s' % (config.data_dir, ext) with benchmark("Tar.%s pack x 10" % ext, format_data["time_pack"]): for i in range(10): with tarfile.open(archive_path, 'w:%s' % ext) as archive: for y in range(100): test_data_io.seek(0) tar_info = tarfile.TarInfo(file_name % y) tar_info.size = 4 * 1024 archive.addfile(tar_info, test_data_io) yield "." hash = CryptHash.sha512sum(open("%s/test.tar.%s" % (config.data_dir, ext), "rb")) valid = format_data["hash"] assert hash == valid, "Invalid hash: %s != %s<br>" % (hash, valid) archive_size = os.path.getsize(archive_path) / 1024 with benchmark("Tar.%s unpack (%.2fkB) x 10" % (ext, archive_size), format_data["time_unpack"]): for i in range(10): with tarfile.open(archive_path, 'r:%s' % ext) as archive: for y in range(100): assert archive.extractfile(file_name % y).read() == test_data yield "." if os.path.isfile(archive_path): os.unlink(archive_path) yield "<br>Done. Total: %.2fs" % (time.time() - t)
def handleGetFile(self, params, streaming=False): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False try: file_path = site.storage.getPath(params["inner_path"]) if streaming: file_obj = site.storage.open(params["inner_path"]) else: file_obj = Msgpack.FilePart(file_path, "rb") with file_obj as file: file.seek(params["location"]) read_bytes = params.get("read_bytes", FILE_BUFF) file_size = os.fstat(file.fileno()).st_size if file_size > read_bytes: # Check if file is readable at current position (for big files) if not self.isReadable(site, params["inner_path"], file, params["location"]): raise RequestError( "File not readable at position: %s" % params["location"]) else: if params.get( "file_size") and params["file_size"] != file_size: self.connection.badAction(2) raise RequestError( "File size does not match: %sB != %sB" % (params["file_size"], file_size)) if not streaming: file.read_bytes = read_bytes if params["location"] > file_size: self.connection.badAction(5) raise RequestError("Bad file location") if streaming: back = { "size": file_size, "location": min(file.tell() + read_bytes, file_size), "stream_bytes": min(read_bytes, file_size - params["location"]) } self.response(back) self.sendRawfile(file, read_bytes=read_bytes) else: back = { "body": file, "size": file_size, "location": min(file.tell() + file.read_bytes, file_size) } self.response(back, streaming=True) bytes_sent = min( read_bytes, file_size - params["location"]) # Number of bytes we going to send site.settings["bytes_sent"] = site.settings.get( "bytes_sent", 0) + bytes_sent if config.debug_socket: self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent)) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect( self.connection) # Assign current connection to peer return { "bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"] } except RequestError as err: self.log.debug("GetFile %s %s request error: %s" % (self.connection, params["inner_path"], Debug.formatException(err))) self.response({"error": "File read error: %s" % err}) except OSError as err: if config.verbose: self.log.debug("GetFile read error: %s" % Debug.formatException(err)) self.response({"error": "File read error"}) return False except Exception as err: self.log.error("GetFile exception: %s" % Debug.formatException(err)) self.response({"error": "File read exception"}) return False
def handleBigfileUpload(self, upload_info, read): site = upload_info["site"] inner_path = upload_info["inner_path"] with site.storage.open(inner_path, "wb", create_dirs=True) as out_file: merkle_root, piece_size, piecemap_info = site.content_manager.hashBigfile( read, upload_info["size"], upload_info["piece_size"], out_file) if len(piecemap_info["sha512_pieces"]) == 1: # Small file, don't split hash = binascii.hexlify(piecemap_info["sha512_pieces"][0]) hash_id = site.content_manager.hashfield.getHashId(hash) site.content_manager.optionalDownloaded(inner_path, hash_id, upload_info["size"], own=True) else: # Big file file_name = helper.getFilename(inner_path) site.storage.open(upload_info["piecemap"], "wb").write( Msgpack.pack({file_name: piecemap_info})) # Find piecemap and file relative path to content.json file_info = site.content_manager.getFileInfo(inner_path, new_file=True) content_inner_path_dir = helper.getDirname( file_info["content_inner_path"]) piecemap_relative_path = upload_info["piecemap"][ len(content_inner_path_dir):] file_relative_path = inner_path[len(content_inner_path_dir):] # Add file to content.json if site.storage.isFile(file_info["content_inner_path"]): content = site.storage.loadJson( file_info["content_inner_path"]) else: content = {} if "files_optional" not in content: content["files_optional"] = {} content["files_optional"][file_relative_path] = { "sha512": merkle_root, "size": upload_info["size"], "piecemap": piecemap_relative_path, "piece_size": piece_size } merkle_root_hash_id = site.content_manager.hashfield.getHashId( merkle_root) site.content_manager.optionalDownloaded(inner_path, merkle_root_hash_id, upload_info["size"], own=True) site.storage.writeJson(file_info["content_inner_path"], content) site.content_manager.contents.loadItem( file_info["content_inner_path"]) # reload cache return json.dumps({ "merkle_root": merkle_root, "piece_num": len(piecemap_info["sha512_pieces"]), "piece_size": piece_size, "inner_path": inner_path })
def hashFile(self, dir_inner_path, file_relative_path, optional=False): inner_path = dir_inner_path + file_relative_path file_size = self.site.storage.getSize(inner_path) # Only care about optional files >1MB if not optional or file_size < 1 * 1024 * 1024: return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional) back = {} content = self.contents.get(dir_inner_path + "content.json") hash = None piecemap_relative_path = None piece_size = None # Don't re-hash if it's already in content.json if content and file_relative_path in content.get("files_optional", {}): file_node = content["files_optional"][file_relative_path] if file_node["size"] == file_size: self.log.info("- [SAME SIZE] %s" % file_relative_path) hash = file_node.get("sha512") piecemap_relative_path = file_node.get("piecemap") piece_size = file_node.get("piece_size") if not hash or not piecemap_relative_path: # Not in content.json yet if file_size < 5 * 1024 * 1024: # Don't create piecemap automatically for files smaller than 5MB return super(ContentManagerPlugin, self).hashFile(dir_inner_path, file_relative_path, optional) self.log.info("- [HASHING] %s" % file_relative_path) merkle_root, piece_size, piecemap_info = self.hashBigfile( self.site.storage.open(inner_path, "rb").read, file_size) if not hash: hash = merkle_root if not piecemap_relative_path: file_name = helper.getFilename(file_relative_path) piecemap_relative_path = file_relative_path + ".piecemap.msgpack" piecemap_inner_path = inner_path + ".piecemap.msgpack" self.site.storage.open(piecemap_inner_path, "wb").write( Msgpack.pack({file_name: piecemap_info})) back.update( super(ContentManagerPlugin, self).hashFile(dir_inner_path, piecemap_relative_path, optional=True)) piece_num = int(math.ceil(float(file_size) / piece_size)) # Add the merkle root to hashfield hash_id = self.site.content_manager.hashfield.getHashId(hash) self.optionalDownloaded(inner_path, hash_id, file_size, own=True) self.site.storage.piecefields[hash].frombytes(b"\x01" * piece_num) back[file_relative_path] = { "sha512": hash, "size": file_size, "piecemap": piecemap_relative_path, "piece_size": piece_size } return back