def testStreaming(self): f = StreamingMsgpack.FilePart("%s/users.json" % config.data_dir) f.read_bytes = 10 data = {"cmd": "response", "params": f} out_buff = StringIO.StringIO() StreamingMsgpack.stream(data, out_buff.write) out_buff.seek(0) data_packb = {"cmd": "response", "params": open("%s/users.json" % config.data_dir).read(10)} out_buff.seek(0) assert msgpack.unpackb(out_buff.read()) == data_packb
def send(self, message, streaming=False): self.last_send_time = time.time() if config.debug_socket: self.log( "Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) if not self.sock: self.log("Send error: missing socket") return False try: if streaming: with self.send_lock: bytes_sent = StreamingMsgpack.stream( message, self.sock.sendall) message = None self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent else: data = msgpack.packb(message) message = None self.bytes_sent += len(data) self.server.bytes_sent += len(data) with self.send_lock: self.sock.sendall(data) except Exception, err: self.close("Send error: %s" % err) return False
def send(self, message, streaming=False): if config.debug_socket: self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % ( message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id")) ) if not self.sock: self.log("Send error: missing socket") return False self.last_send_time = time.time() try: if streaming: bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall) message = None self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent else: data = msgpack.packb(message) message = None self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.sock.sendall(data) except Exception, err: self.close("Send error: %s" % err) return False
def actionGetFile(self, params): site = self.sites.get(params["site"]) if not site or not site.settings["serving"]: # Site unknown or not serving self.response({"error": "Unknown site"}) return False try: file_path = site.storage.getPath(params["inner_path"]) if config.debug_socket: self.log.debug("Opening file: %s" % file_path) with StreamingMsgpack.FilePart(file_path, "rb") as file: file.seek(params["location"]) file.read_bytes = FILE_BUFF back = {} back["body"] = file back["size"] = os.fstat(file.fileno()).st_size back["location"] = min(file.tell()+FILE_BUFF, back["size"]) if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"])) self.response(back, streaming=True) if config.debug_socket: self.log.debug("File %s sent" % file_path) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port) if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer except Exception, err: self.log.debug("GetFile read error: %s" % Debug.formatException(err)) self.response({"error": "File read error: %s" % Debug.formatException(err)}) return False
def testStreaming(self): f = StreamingMsgpack.FilePart("%s/users.json" % config.data_dir) f.read_bytes = 10 data = {"cmd": "response", "params": f} out_buff = StringIO.StringIO() StreamingMsgpack.stream(data, out_buff.write) out_buff.seek(0) data_packb = { "cmd": "response", "params": open("%s/users.json" % config.data_dir).read(10) } out_buff.seek(0) assert msgpack.unpackb(out_buff.read()) == data_packb
def actionGetFile(self, params): site = self.sites.get(params["site"]) if not site or not site.settings[ "serving"]: # Site unknown or not serving self.response({"error": "Unknown site"}) return False try: file_path = site.storage.getPath(params["inner_path"]) with StreamingMsgpack.FilePart(file_path, "rb") as file: file.seek(params["location"]) file.read_bytes = FILE_BUFF file_size = os.fstat(file.fileno()).st_size if params.get( "file_size") and params["file_size"] != file_size: self.connection.badAction(5) raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size)) if params["location"] > file_size: self.connection.badAction(5) raise RequestError("Bad file location") back = { "body": file, "size": file_size, "location": min(file.tell() + FILE_BUFF, file_size) } self.response(back, streaming=True) bytes_sent = min( FILE_BUFF, file_size - params["location"]) # Number of bytes we going to send site.settings["bytes_sent"] = site.settings.get( "bytes_sent", 0) + bytes_sent if config.debug_socket: self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent)) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port) if connected_peer: # Just added connected_peer.connect( self.connection) # Assign current connection to peer return { "bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"] } except RequestError, err: self.log.debug("GetFile %s %s request error: %s" % (self.connection, params["inner_path"], Debug.formatException(err))) self.response({"error": "File read error: %s" % err})
def send(self, message, streaming=False): if config.debug_socket: self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) self.last_send_time = time.time() if streaming: bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall) message = None self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent else: data = msgpack.packb(message) message = None self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.sock.sendall(data) self.last_sent_time = time.time() return True
def send(self, message, streaming=False): self.last_send_time = time.time() if config.debug_socket: self.log( "Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) if not self.sock: self.log("Send error: missing socket") return False if not self.connected and message.get("cmd") != "handshake": self.log("Wait for handshake before send request") self.event_connected.get() try: stat_key = message.get("cmd", "unknown") if stat_key == "response": stat_key = "response: %s" % self.last_cmd_recv else: self.server.num_sent += 1 self.server.stat_sent[stat_key]["num"] += 1 if streaming: with self.send_lock: bytes_sent = StreamingMsgpack.stream( message, self.sock.sendall) self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent self.server.stat_sent[stat_key]["bytes"] += bytes_sent message = None else: data = msgpack.packb(message) self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.server.stat_sent[stat_key]["bytes"] += len(data) message = None with self.send_lock: self.sock.sendall(data) except Exception, err: self.close("Send error: %s (cmd: %s)" % (err, stat_key)) return False
def send(self, message, streaming=False): self.last_send_time = time.time() if config.debug_socket: self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % ( message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id")) ) if not self.sock: self.log("Send error: missing socket") return False if not self.connected and message.get("cmd") != "handshake": self.log("Wait for handshake before send request") self.event_connected.get() try: stat_key = message.get("cmd", "unknown") if stat_key == "response": stat_key = "response: %s" % self.last_cmd_recv else: self.server.num_sent += 1 self.server.stat_sent[stat_key]["num"] += 1 if streaming: with self.send_lock: bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall) self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent self.server.stat_sent[stat_key]["bytes"] += bytes_sent message = None else: data = msgpack.packb(message) self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.server.stat_sent[stat_key]["bytes"] += len(data) message = None with self.send_lock: self.sock.sendall(data) except Exception, err: self.close("Send error: %s (cmd: %s)" % (err, stat_key)) return False
def send(self, message, streaming=False): if config.debug_socket: self.log( "Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id"))) self.last_send_time = time.time() if streaming: bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall) message = None self.bytes_sent += bytes_sent self.server.bytes_sent += bytes_sent else: data = msgpack.packb(message) message = None self.bytes_sent += len(data) self.server.bytes_sent += len(data) self.sock.sendall(data) self.last_sent_time = time.time() return True
def handleGetFile(self, params, streaming=False): site = self.sites.get(params["site"]) if not site or not site.settings["serving"]: # Site unknown or not serving self.response({"error": "Unknown site"}) return False try: file_path = site.storage.getPath(params["inner_path"]) if streaming: file_obj = site.storage.open(params["inner_path"]) else: file_obj = StreamingMsgpack.FilePart(file_path, "rb") with file_obj as file: file.seek(params["location"]) read_bytes = params.get("read_bytes", FILE_BUFF) file_size = os.fstat(file.fileno()).st_size if file_size > read_bytes: # Check if file is readable at current position (for big files) if not self.isReadable(site, params["inner_path"], file, params["location"]): raise RequestError("File not readable at position: %s" % params["location"]) if not streaming: file.read_bytes = read_bytes if params.get("file_size") and params["file_size"] != file_size: self.connection.badAction(2) raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size)) if params["location"] > file_size: self.connection.badAction(5) raise RequestError("Bad file location") if streaming: back = { "size": file_size, "location": min(file.tell() + read_bytes, file_size), "stream_bytes": min(read_bytes, file_size - params["location"]) } self.response(back) self.sendRawfile(file, read_bytes=read_bytes) else: back = { "body": file, "size": file_size, "location": min(file.tell() + file.read_bytes, file_size) } self.response(back, streaming=True) bytes_sent = min(read_bytes, file_size - params["location"]) # Number of bytes we going to send site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent if config.debug_socket: self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent)) # Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer return {"bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"]} except RequestError, err: self.log.debug("GetFile %s %s request error: %s" % (self.connection, params["inner_path"], Debug.formatException(err))) self.response({"error": "File read error: %s" % err})