def testStreaming(self):
f = StreamingMsgpack.FilePart("%s/users.json" % config.data_dir)
f.read_bytes = 10
data = {"cmd": "response", "params": f}
out_buff = StringIO.StringIO()
StreamingMsgpack.stream(data, out_buff.write)
out_buff.seek(0)
data_packb = {"cmd": "response", "params": open("%s/users.json" % config.data_dir).read(10)}
out_buff.seek(0)
assert msgpack.unpackb(out_buff.read()) == data_packb
def send(self, message, streaming=False):
self.last_send_time = time.time()
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
% (message.get("cmd"), message.get("to"), streaming,
message.get("params", {}).get("site"),
message.get("params",
{}).get("inner_path"), message.get("req_id")))
if not self.sock:
self.log("Send error: missing socket")
return False
try:
if streaming:
with self.send_lock:
bytes_sent = StreamingMsgpack.stream(
message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
with self.send_lock:
self.sock.sendall(data)
except Exception, err:
self.close("Send error: %s" % err)
return False
def send(self, message, streaming=False):
if config.debug_socket:
self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (
message.get("cmd"), message.get("to"), streaming,
message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"),
message.get("req_id"))
)
if not self.sock:
self.log("Send error: missing socket")
return False
self.last_send_time = time.time()
try:
if streaming:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
except Exception, err:
self.close("Send error: %s" % err)
return False
def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
if config.debug_socket: self.log.debug("Opening file: %s" % file_path)
with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"])
file.read_bytes = FILE_BUFF
back = {}
back["body"] = file
back["size"] = os.fstat(file.fileno()).st_size
back["location"] = min(file.tell()+FILE_BUFF, back["size"])
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
self.response(back, streaming=True)
if config.debug_socket: self.log.debug("File %s sent" % file_path)
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def testStreaming(self):
f = StreamingMsgpack.FilePart("%s/users.json" % config.data_dir)
f.read_bytes = 10
data = {"cmd": "response", "params": f}
out_buff = StringIO.StringIO()
StreamingMsgpack.stream(data, out_buff.write)
out_buff.seek(0)
data_packb = {
"cmd": "response",
"params": open("%s/users.json" % config.data_dir).read(10)
}
out_buff.seek(0)
assert msgpack.unpackb(out_buff.read()) == data_packb
def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings[
"serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"])
file.read_bytes = FILE_BUFF
file_size = os.fstat(file.fileno()).st_size
if params.get(
"file_size") and params["file_size"] != file_size:
self.connection.badAction(5)
raise RequestError("File size does not match: %sB != %sB" %
(params["file_size"], file_size))
if params["location"] > file_size:
self.connection.badAction(5)
raise RequestError("Bad file location")
back = {
"body": file,
"size": file_size,
"location": min(file.tell() + FILE_BUFF, file_size)
}
self.response(back, streaming=True)
bytes_sent = min(
FILE_BUFF, file_size -
params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get(
"bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s at position %s sent %s bytes" %
(file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip,
self.connection.port)
if connected_peer: # Just added
connected_peer.connect(
self.connection) # Assign current connection to peer
return {
"bytes_sent": bytes_sent,
"file_size": file_size,
"location": params["location"]
}
except RequestError, err:
self.log.debug("GetFile %s %s request error: %s" %
(self.connection, params["inner_path"],
Debug.formatException(err)))
self.response({"error": "File read error: %s" % err})
def send(self, message, streaming=False):
if config.debug_socket: self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (message.get("cmd"), message.get("to"), streaming, message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"), message.get("req_id")))
self.last_send_time = time.time()
if streaming:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
self.last_sent_time = time.time()
return True
def send(self, message, streaming=False):
self.last_send_time = time.time()
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
% (message.get("cmd"), message.get("to"), streaming,
message.get("params", {}).get("site"),
message.get("params",
{}).get("inner_path"), message.get("req_id")))
if not self.sock:
self.log("Send error: missing socket")
return False
if not self.connected and message.get("cmd") != "handshake":
self.log("Wait for handshake before send request")
self.event_connected.get()
try:
stat_key = message.get("cmd", "unknown")
if stat_key == "response":
stat_key = "response: %s" % self.last_cmd_recv
else:
self.server.num_sent += 1
self.server.stat_sent[stat_key]["num"] += 1
if streaming:
with self.send_lock:
bytes_sent = StreamingMsgpack.stream(
message, self.sock.sendall)
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
self.server.stat_sent[stat_key]["bytes"] += bytes_sent
message = None
else:
data = msgpack.packb(message)
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.server.stat_sent[stat_key]["bytes"] += len(data)
message = None
with self.send_lock:
self.sock.sendall(data)
except Exception, err:
self.close("Send error: %s (cmd: %s)" % (err, stat_key))
return False
def send(self, message, streaming=False):
self.last_send_time = time.time()
if config.debug_socket:
self.log("Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s" % (
message.get("cmd"), message.get("to"), streaming,
message.get("params", {}).get("site"), message.get("params", {}).get("inner_path"),
message.get("req_id"))
)
if not self.sock:
self.log("Send error: missing socket")
return False
if not self.connected and message.get("cmd") != "handshake":
self.log("Wait for handshake before send request")
self.event_connected.get()
try:
stat_key = message.get("cmd", "unknown")
if stat_key == "response":
stat_key = "response: %s" % self.last_cmd_recv
else:
self.server.num_sent += 1
self.server.stat_sent[stat_key]["num"] += 1
if streaming:
with self.send_lock:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
self.server.stat_sent[stat_key]["bytes"] += bytes_sent
message = None
else:
data = msgpack.packb(message)
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.server.stat_sent[stat_key]["bytes"] += len(data)
message = None
with self.send_lock:
self.sock.sendall(data)
except Exception, err:
self.close("Send error: %s (cmd: %s)" % (err, stat_key))
return False
def send(self, message, streaming=False):
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
% (message.get("cmd"), message.get("to"), streaming,
message.get("params", {}).get("site"),
message.get("params",
{}).get("inner_path"), message.get("req_id")))
self.last_send_time = time.time()
if streaming:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
self.last_sent_time = time.time()
return True
def handleGetFile(self, params, streaming=False):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
if streaming:
file_obj = site.storage.open(params["inner_path"])
else:
file_obj = StreamingMsgpack.FilePart(file_path, "rb")
with file_obj as file:
file.seek(params["location"])
read_bytes = params.get("read_bytes", FILE_BUFF)
file_size = os.fstat(file.fileno()).st_size
if file_size > read_bytes: # Check if file is readable at current position (for big files)
if not self.isReadable(site, params["inner_path"], file, params["location"]):
raise RequestError("File not readable at position: %s" % params["location"])
if not streaming:
file.read_bytes = read_bytes
if params.get("file_size") and params["file_size"] != file_size:
self.connection.badAction(2)
raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size))
if params["location"] > file_size:
self.connection.badAction(5)
raise RequestError("Bad file location")
if streaming:
back = {
"size": file_size,
"location": min(file.tell() + read_bytes, file_size),
"stream_bytes": min(read_bytes, file_size - params["location"])
}
self.response(back)
self.sendRawfile(file, read_bytes=read_bytes)
else:
back = {
"body": file,
"size": file_size,
"location": min(file.tell() + file.read_bytes, file_size)
}
self.response(back, streaming=True)
bytes_sent = min(read_bytes, file_size - params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request")
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
return {"bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"]}
except RequestError, err:
self.log.debug("GetFile %s %s request error: %s" % (self.connection, params["inner_path"], Debug.formatException(err)))
self.response({"error": "File read error: %s" % err})