def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
if config.debug_socket: self.log.debug("Opening file: %s" % file_path)
file = open(file_path, "rb")
file.seek(params["location"])
back = {}
back["body"] = file.read(FILE_BUFF)
back["location"] = file.tell()
back["size"] = os.fstat(file.fileno()).st_size
if config.debug_socket: self.log.debug("Sending file %s from position %s to %s" % (file_path, params["location"], back["location"]))
self.response(back)
if config.debug_socket: self.log.debug("File %s sent" % file_path)
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def actionGetFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
file_path = site.storage.getPath(params["inner_path"])
with StreamingMsgpack.FilePart(file_path, "rb") as file:
file.seek(params["location"])
file.read_bytes = FILE_BUFF
file_size = os.fstat(file.fileno()).st_size
assert params["location"] <= file_size, "Bad file location"
back = {
"body": file,
"size": file_size,
"location": min(file.tell() + FILE_BUFF, file_size)
}
self.response(back, streaming=True)
bytes_sent = min(FILE_BUFF, file_size - params["location"]) # Number of bytes we going to send
site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent
if config.debug_socket:
self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent))
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def actionStreamFile(self, params):
site = self.sites.get(params["site"])
if not site or not site.settings["serving"]: # Site unknown or not serving
self.response({"error": "Unknown site"})
return False
try:
if config.debug_socket:
self.log.debug("Opening file: %s" % params["inner_path"])
with site.storage.open(params["inner_path"]) as file:
file.seek(params["location"])
stream_bytes = min(FILE_BUFF, os.fstat(file.fileno()).st_size-params["location"])
back = {
"size": os.fstat(file.fileno()).st_size,
"location": min(file.tell() + FILE_BUFF, os.fstat(file.fileno()).st_size),
"stream_bytes": stream_bytes
}
if config.debug_socket:
self.log.debug(
"Sending file %s from position %s to %s" %
(params["inner_path"], params["location"], back["location"])
)
self.response(back)
self.sendRawfile(file, read_bytes=FILE_BUFF)
if config.debug_socket:
self.log.debug("File %s sent" % params["inner_path"])
# Add peer to site if not added before
connected_peer = site.addPeer(self.connection.ip, self.connection.port)
if connected_peer: # Just added
connected_peer.connect(self.connection) # Assign current connection to peer
except Exception, err:
self.log.debug("GetFile read error: %s" % Debug.formatException(err))
self.response({"error": "File read error: %s" % Debug.formatException(err)})
return False
def asyncErrorWatcher(func, *args, **kwargs):
try:
func(*args, **kwargs)
except Exception, err:
if config.debug: # Allow websocket errors to appear on /Debug
sys.modules["main"].DebugHook.handleError()
self.log.error("WebSocket handleRequest error: %s" % Debug.formatException(err))
self.cmd("error", "Internal error: %s" % Debug.formatException(err, "html"))
def actionSiteSign(self, to, privatekey=None, inner_path="content.json", remove_missing_optional=False, update_changed_files=False, response_ok=True):
self.log.debug("Signing: %s" % inner_path)
site = self.site
extend = {} # Extended info for signing
# Change to the file's content.json
file_info = site.content_manager.getFileInfo(inner_path)
if not inner_path.endswith("content.json"):
if not file_info:
raise Exception("Invalid content.json file: %s" % inner_path)
inner_path = file_info["content_inner_path"]
# Add certificate to user files
is_user_content = file_info and ("cert_signers" in file_info or "cert_signers_pattern" in file_info)
if is_user_content and privatekey is None:
cert = self.user.getCert(self.site.address)
extend["cert_auth_type"] = cert["auth_type"]
extend["cert_user_id"] = self.user.getCertUserId(site.address)
extend["cert_sign"] = cert["cert_sign"]
self.log.debug("Extending content.json with cert %s" % extend["cert_user_id"])
if not self.hasFilePermission(inner_path):
self.log.error("SiteSign error: you don't own this site & site owner doesn't allow you to do so.")
return self.response(to, {"error": "Forbidden, you can only modify your own sites"})
if privatekey == "stored": # Get privatekey from sites.json
privatekey = self.user.getSiteData(self.site.address).get("privatekey")
if not privatekey: # Get privatekey from users.json auth_address
privatekey = self.user.getAuthPrivatekey(self.site.address)
# Signing
# Reload content.json, ignore errors to make it up-to-date
site.content_manager.loadContent(inner_path, add_bad_files=False, force=True)
# Sign using private key sent by user
try:
site.content_manager.sign(inner_path, privatekey, extend=extend, update_changed_files=update_changed_files, remove_missing_optional=remove_missing_optional)
except (VerifyError, SignError) as err:
self.cmd("notification", ["error", _["Content signing failed"] + "<br><small>%s</small>" % err])
self.response(to, {"error": "Site sign failed: %s" % err})
self.log.error("Site sign failed: %s: %s" % (inner_path, Debug.formatException(err)))
return
except Exception as err:
self.cmd("notification", ["error", _["Content signing error"] + "<br><small>%s</small>" % Debug.formatException(err)])
self.response(to, {"error": "Site sign error: %s" % Debug.formatException(err)})
self.log.error("Site sign error: %s: %s" % (inner_path, Debug.formatException(err)))
return
site.content_manager.loadContent(inner_path, add_bad_files=False) # Load new content.json, ignore errors
if update_changed_files:
self.site.updateWebsocket(file_done=inner_path)
if response_ok:
self.response(to, "ok")
else:
return inner_path
def publisher(self, inner_path, peers, published, limit, diffs={}, event_done=None, cb_progress=None):
file_size = self.storage.getSize(inner_path)
content_json_modified = self.content_manager.contents[inner_path]["modified"]
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop()
if peer in published:
continue
if peer.last_content_json_update == content_json_modified:
self.log.debug("%s already received this update for %s, skipping" % (peer, inner_path))
continue
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 10sec + size in kb
timeout = 10 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"diffs": diffs
})
if result:
break
except Exception, err:
self.log.error("Publish error: %s" % Debug.formatException(err))
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
if cb_progress and len(published) <= limit:
cb_progress(len(published), limit)
self.log.info("[OK] %s: %s %s/%s" % (peer.key, result["ok"], len(published), limit))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError("Publish timeout")
self.log.info("[FAILED] %s: %s" % (peer.key, result))
time.sleep(0.01)
def createBroadcastSocket(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if hasattr(socket, 'SO_REUSEPORT'):
try:
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
except Exception as err:
self.log.warning("Error setting SO_REUSEPORT: %s" % err)
binded = False
for retry in range(3):
try:
sock.bind((self.listen_ip, self.listen_port))
binded = True
break
except Exception as err:
self.log.error(
"Socket bind to %s:%s error: %s, retry #%s" %
(self.listen_ip, self.listen_port, Debug.formatException(err), retry)
)
time.sleep(retry)
if binded:
return sock
else:
return False
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
content_json_modified = self.content_manager.contents[inner_path]["modified"]
body = self.storage.read(inner_path)
# Find out my ip and port
tor_manager = self.connection_server.tor_manager
if tor_manager.enabled and tor_manager.start_onions:
my_ip = tor_manager.getOnion(self.address)
if my_ip:
my_ip += ".onion"
my_port = config.fileserver_port
else:
my_ip = config.ip_external
if self.connection_server.port_opened:
my_port = config.fileserver_port
else:
my_port = 0
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer in published:
continue
if peer.last_content_json_update == content_json_modified:
self.log.debug("%s already received this update for %s, skipping" % (peer, inner_path))
continue
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 10sec + size in kb
timeout = 10 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (my_ip, my_port)
})
if result:
break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s %s/%s" % (peer.key, result["ok"], len(published), limit))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
def connect(self, connection=None):
if self.reputation < -10:
self.reputation = -10
if self.reputation > 10:
self.reputation = 10
if self.connection:
self.log("Getting connection (Closing %s)..." % self.connection)
self.connection.close("Connection change")
else:
self.log("Getting connection (reputation: %s)..." % self.reputation)
if connection: # Connection specified
self.log("Assigning connection %s" % connection)
self.connection = connection
self.connection.sites += 1
else: # Try to find from connection pool or create new connection
self.connection = None
try:
if self.connection_server:
connection_server = self.connection_server
elif self.site:
connection_server = self.site.connection_server
else:
connection_server = sys.modules["main"].file_server
self.connection = connection_server.getConnection(self.ip, self.port, site=self.site, is_tracker_connection=self.is_tracker_connection)
self.reputation += 1
self.connection.sites += 1
except Exception, err:
self.onConnectionError("Getting connection error")
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None
def actionFileWrite(self, to, inner_path, content_base64):
if (
not self.site.settings["own"] and
self.user.getAuthAddress(self.site.address) not in self.site.content_manager.getValidSigners(inner_path)
):
return self.response(to, {"error": "Forbidden, you can only modify your own files"})
try:
import base64
content = base64.b64decode(content_base64)
# Save old file to generate patch later
if (
inner_path.endswith(".json") and not inner_path.endswith("content.json") and
self.site.storage.isFile(inner_path) and not self.site.storage.isFile(inner_path + "-old")
):
try:
self.site.storage.rename(inner_path, inner_path + "-old")
except Exception:
# Rename failed, fall back to standard file write
f_old = self.site.storage.open(inner_path, "rb")
f_new = self.site.storage.open(inner_path + "-old", "wb")
shutil.copyfileobj(f_old, f_new)
self.site.storage.write(inner_path, content)
except Exception, err:
return self.response(to, {"error": "Write error: %s" % Debug.formatException(err)})
def request(self, cmd, params={}, stream_to=None):
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
self.onConnectionError()
return None # Connection failed
self.log("Send request: %s %s" % (params.get("site", ""), cmd))
for retry in range(1, 4): # Retry 3 times
try:
res = self.connection.request(cmd, params, stream_to)
if not res:
raise Exception("Send error")
if "error" in res:
self.log("%s error: %s" % (cmd, res["error"]))
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
self.time_response = time.time()
return res
except Exception, err:
if type(err).__name__ == "Notify": # Greenlet killed by worker
self.log("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
break
else:
self.onConnectionError()
self.log(
"%s (connection_error: %s, hash_failed: %s, retry: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed, retry)
)
time.sleep(1 * retry)
self.connect()
def start(self): # Listens for discover requests
self.sock = self.createBroadcastSocket()
if not self.sock:
self.log.error("Unable to listen on port %s" % self.listen_port)
return
self.log.debug("Started on port %s" % self.listen_port)
self.running = True
while self.running:
try:
data, addr = self.sock.recvfrom(8192)
except Exception as err:
if self.running:
self.log.error("Listener receive error: %s" % err)
continue
if not self.running:
break
try:
message = msgpack.unpackb(data)
response_addr, message = self.handleMessage(addr, message)
if message:
self.send(response_addr, message)
except Exception as err:
self.log.error("Handlemessage error: %s" % Debug.formatException(err))
self.log.debug("Stopped listening on port %s" % self.listen_port)
def messageLoop(self):
if not self.sock:
self.log("Socket error: No socket found")
return False
self.protocol = "v2"
self.updateName()
self.connected = True
self.unpacker = msgpack.Unpacker()
sock = self.sock
try:
while True:
buff = sock.recv(16*1024)
if not buff: break # Connection closed
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += len(buff)
self.server.bytes_recv += len(buff)
if not self.unpacker:
self.unpacker = msgpack.Unpacker()
self.unpacker.feed(buff)
for message in self.unpacker:
self.incomplete_buff_recv = 0
self.handleMessage(message)
message = None
buff = None
except Exception, err:
if not self.closed: self.log("Socket error: %s" % Debug.formatException(err))
def send(self, message, streaming=False):
if config.debug_socket:
self.log(
"Send: %s, to: %s, streaming: %s, site: %s, inner_path: %s, req_id: %s"
% (
message.get("cmd"),
message.get("to"),
streaming,
message.get("params", {}).get("site"),
message.get("params", {}).get("inner_path"),
message.get("req_id"),
)
)
self.last_send_time = time.time()
try:
if streaming:
bytes_sent = StreamingMsgpack.stream(message, self.sock.sendall)
message = None
self.bytes_sent += bytes_sent
self.server.bytes_sent += bytes_sent
else:
data = msgpack.packb(message)
message = None
self.bytes_sent += len(data)
self.server.bytes_sent += len(data)
self.sock.sendall(data)
except Exception, err:
self.log("Send errror: %s" % Debug.formatException(err))
self.close()
return False
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
timeout = timeout = 5+int(file_size/1024)+peer.connection.last_ping_delay # Timeout: 5sec + size in kb + last_ping
else:
timeout = timeout = 5+int(file_size/1024) # Timeout: 5sec + size in kb
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}: peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
def actionSidebarGetPeers(self, to):
permissions = self.getPermissions(to)
if "ADMIN" not in permissions:
return self.response(to, "You don't have permission to run this command")
try:
peer_locations = self.getPeerLocations(self.site.peers)
globe_data = []
ping_times = [
peer_location["ping"]
for peer_location in peer_locations
if peer_location["ping"]
]
if ping_times:
ping_avg = sum(ping_times) / float(len(ping_times))
else:
ping_avg = 0
for peer_location in peer_locations:
if peer_location["ping"] == 0: # Me
height = -0.135
elif peer_location["ping"]:
height = min(0.20, math.log(1 + peer_location["ping"] / ping_avg, 300))
else:
height = -0.03
globe_data += [peer_location["lat"], peer_location["lon"], height]
self.response(to, globe_data)
except Exception, err:
self.log.debug("sidebarGetPeers error: %s" % Debug.formatException(err))
self.response(to, {"error": err})
def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False):
from Site import Site
from Site import SiteManager
from Debug import Debug
SiteManager.site_manager.load()
logging.info("Signing site: %s..." % address)
site = Site(address, allow_create=False)
if not privatekey: # If no privatekey defined
from User import UserManager
user = UserManager.user_manager.get()
if user:
site_data = user.getSiteData(address)
privatekey = site_data.get("privatekey")
else:
privatekey = None
if not privatekey:
# Not found in users.json, ask from console
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
diffs = site.content_manager.getDiffs(inner_path)
try:
succ = site.content_manager.sign(inner_path=inner_path, privatekey=privatekey, update_changed_files=True, remove_missing_optional=remove_missing_optional)
except Exception, err:
logging.error("Sign error: %s" % Debug.formatException(err))
succ = False
def connect(self, connection=None):
if self.connection:
self.log("Getting connection (Closing %s)..." % self.connection)
self.connection.close()
else:
self.log("Getting connection...")
if connection: # Connection specified
self.connection = connection
else: # Try to find from connection pool or create new connection
self.connection = None
try:
if self.connection_server:
self.connection = self.connection_server.getConnection(self.ip, self.port, site=self.site)
elif self.site:
self.connection = self.site.connection_server.getConnection(self.ip, self.port, site=self.site)
else:
self.connection = sys.modules["main"].file_server.getConnection(self.ip, self.port, site=self.site)
except Exception, err:
self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None
def startTor(self):
if sys.platform.startswith("win"):
try:
if not os.path.isfile(self.tor_exe):
self.downloadTor()
self.log.info("Starting Tor client %s..." % self.tor_exe)
tor_dir = os.path.dirname(self.tor_exe)
startupinfo = subprocess.STARTUPINFO()
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
cmd = r"%s -f torrc --defaults-torrc torrc-defaults --ignore-missing-torrc" % self.tor_exe
if config.tor_use_bridges:
cmd += " --UseBridges 1"
self.tor_process = subprocess.Popen(cmd, cwd=tor_dir, close_fds=True, startupinfo=startupinfo)
for wait in range(1, 10): # Wait for startup
time.sleep(wait * 0.5)
self.enabled = True
if self.connect():
if self.isSubprocessRunning():
self.request("TAKEOWNERSHIP") # Shut down Tor client when controll connection closed
break
# Terminate on exit
atexit.register(self.stopTor)
except Exception, err:
self.log.error(u"Error starting Tor client: %s" % Debug.formatException(str(err).decode("utf8", "ignore")))
self.enabled = False
def request(self, cmd, params = {}):
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
self.onConnectionError()
return None # Connection failed
#if cmd != "ping" and self.last_response and time.time() - self.last_response > 20*60: # If last response if older than 20 minute, ping first to see if still alive
# if not self.ping(): return None
for retry in range(1,3): # Retry 3 times
#if config.debug_socket: self.log.debug("sendCmd: %s %s" % (cmd, params.get("inner_path")))
try:
response = self.connection.request(cmd, params)
if not response: raise Exception("Send error")
#if config.debug_socket: self.log.debug("Got response to: %s" % cmd)
if "error" in response:
self.log.debug("%s error: %s" % (cmd, response["error"]))
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
self.last_response = time.time()
return response
except Exception, err:
if type(err).__name__ == "Notify": # Greenlet kill by worker
self.log.debug("Peer worker got killed: %s, aborting cmd: %s" % (err.message, cmd))
break
else:
self.onConnectionError()
self.log.debug("%s (connection_error: %s, hash_failed: %s, retry: %s)" % (Debug.formatException(err), self.connection_error, self.hash_failed, retry))
time.sleep(1*retry)
self.connect()
def openport(self, port=None, check=True):
if not port:
port = self.port
if self.port_opened:
return True # Port already opened
if check: # Check first if its already opened
time.sleep(1) # Wait for port open
if self.testOpenport(port, use_alternative=False)["result"] is True:
return True # Port already opened
if config.tor == "always": # Port opening won't work in Tor mode
return False
self.log.info("Trying to open port using UpnpPunch...")
try:
UpnpPunch.ask_to_open_port(self.port, 'ZeroNet', retries=3, protos=["TCP"])
except (UpnpPunch.UpnpError, UpnpPunch.IGDError, socket.error) as err:
self.log.error("UpnpPunch run error: %s" %
Debug.formatException(err))
return False
if self.testOpenport(port)["result"] is True:
self.upnp_port_opened = True
return True
self.log.info("Upnp mapping failed :( Please forward port %s on your router to your ipaddress" % port)
return False
def testOpenportPortchecker(self, port = None):
self.log.info("Checking port %s using portchecker.co..." % port)
try:
data = urllib2.urlopen("http://portchecker.co/check", "port=%s" % port, timeout=20.0).read()
message = re.match('.*<div id="results-wrapper">(.*?)</div>', data, re.DOTALL).group(1)
message = re.sub("<.*?>", "", message.replace("<br>", " ").replace(" ", " ").strip()) # Strip http tags
except Exception, err:
message = "Error: %s" % Debug.formatException(err)
def handleIncomingConnection(self, sock):
self.type = "in"
try:
firstchar = sock.recv(1) # Find out if pure socket or zeromq
except Exception, err:
self.log("Socket firstchar error: %s" % Debug.formatException(err))
self.close()
return False
def messageLoop(self, firstchar=None):
sock = self.sock
try:
if not firstchar: firstchar = sock.recv(1)
except Exception, err:
self.log("Socket firstchar error: %s" % Debug.formatException(err))
self.close()
return False
def testOpenportCanyouseeme(self, port=None):
self.log.info("Checking port %s using canyouseeme.org..." % port)
try:
data = urllib2.urlopen("http://www.canyouseeme.org/", "port=%s" % port, timeout=20.0).read()
message = re.match('.*<p style="padding-left:15px">(.*?)</p>', data, re.DOTALL).group(1)
message = re.sub("<.*?>", "", message.replace("<br>", " ").replace(" ", " ")) # Strip http tags
except Exception, err:
message = "Error: %s" % Debug.formatException(err)
def ping(self):
s = time.time()
response = None
with gevent.Timeout(10.0, False):
try:
response = self.request("ping")
except Exception, err:
self.log("Ping error: %s" % Debug.formatException(err))
def reloadPlugins(self):
self.plugins = {} # Reset registered plugins
for module_name, module in sys.modules.items():
if module and "__file__" in dir(module) and self.plugin_path in module.__file__: # Module file within plugin_path
if "allow_reload" not in dir(module) or module.allow_reload: # Check if reload disabled
try:
reload(module)
except Exception, err:
self.log.error("Plugin %s reload error: %s" % (module_name, Debug.formatException(err)))
def verifyFile(self, inner_path, file, ignore_same=True):
if inner_path.endswith("content.json"): # content.json: Check using sign
from Crypt import CryptBitcoin
try:
new_content = json.load(file)
if inner_path in self.contents:
old_content = self.contents.get(inner_path)
# Checks if its newer the ours
if old_content["modified"] == new_content["modified"] and ignore_same: # Ignore, have the same content.json
return None
elif old_content["modified"] > new_content["modified"]: # We have newer
self.log.debug("We have newer %s (Our: %s, Sent: %s)" % (inner_path, old_content["modified"], new_content["modified"]))
gevent.spawn(self.site.publish, inner_path=inner_path) # Try to fix the broken peers
return False
if new_content["modified"] > time.time() + 60 * 60 * 24: # Content modified in the far future (allow 1 day window)
self.log.error("%s modify is in the future!" % inner_path)
return False
# Check sign
sign = new_content.get("sign")
signs = new_content.get("signs", {})
if "sign" in new_content:
del(new_content["sign"]) # The file signed without the sign
if "signs" in new_content:
del(new_content["signs"]) # The file signed without the signs
sign_content = json.dumps(new_content, sort_keys=True) # Dump the json to string to remove whitepsace
if not self.validContent(inner_path, new_content):
return False # Content not valid (files too large, invalid files)
if signs: # New style signing
valid_signers = self.getValidSigners(inner_path, new_content)
signs_required = self.getSignsRequired(inner_path, new_content)
if inner_path == "content.json" and len(valid_signers) > 1: # Check signers_sign on root content.json
if not CryptBitcoin.verify("%s:%s" % (signs_required, ",".join(valid_signers)), self.site.address, new_content["signers_sign"]):
self.log.error("%s invalid signers_sign!" % inner_path)
return False
if inner_path != "content.json" and not self.verifyCert(inner_path, new_content): # Check if cert valid
self.log.error("%s invalid cert!" % inner_path)
return False
valid_signs = 0
for address in valid_signers:
if address in signs:
valid_signs += CryptBitcoin.verify(sign_content, address, signs[address])
if valid_signs >= signs_required:
break # Break if we has enough signs
return valid_signs >= signs_required
else: # Old style signing
return CryptBitcoin.verify(sign_content, self.site.address, sign)
except Exception, err:
self.log.error("Verify sign error: %s" % Debug.formatException(err))
return False
def start(self):
self.log.info("Starting gshift...")
try:
if platform == 'win32':
command = ['gshift.exe', '--shh']
self.process = Popen(command, stdout=self.fnull, stderr=self.fnull, shell=False)
else:
command = [abspath('gshift'), '--shh']
self.process = Popen(command, stdout=self.fnull, stderr=self.fnull, shell=True)
atexit.register(self.stop)
import time
time.sleep(3)
return self.process
except Exception, err:
print "Error starting gshift: %s" % Debug.formatException(err)
self.log.error("Error starting gshift: %s" % Debug.formatException(err))
return False
def messageLoop(self):
if not self.sock:
self.log("Socket error: No socket found")
return False
self.protocol = "v2"
self.updateName()
self.connected = True
buff_len = 0
req_len = 0
self.unpacker = msgpack.fallback.Unpacker() # Due memory problems of C version
try:
while not self.closed:
buff = self.sock.recv(64 * 1024)
if not buff:
break # Connection closed
buff_len = len(buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
req_len += buff_len
if not self.unpacker:
self.unpacker = msgpack.fallback.Unpacker()
self.unpacker.feed(buff)
buff = None
for message in self.unpacker:
try:
# Stats
self.incomplete_buff_recv = 0
stat_key = message.get("cmd", "unknown")
if stat_key == "response" and "to" in message:
cmd_sent = self.waiting_requests.get(message["to"], {"cmd": "unknown"})["cmd"]
stat_key = "response: %s" % cmd_sent
self.server.stat_recv[stat_key]["bytes"] += req_len
self.server.stat_recv[stat_key]["num"] += 1
if "stream_bytes" in message:
self.server.stat_recv[stat_key]["bytes"] += message["stream_bytes"]
req_len = 0
# Handle message
if "stream_bytes" in message:
self.handleStream(message)
else:
self.handleMessage(message)
except TypeError:
raise Exception("Invalid message type: %s" % type(message))
message = None
except Exception, err:
if not self.closed:
self.log("Socket error: %s" % Debug.formatException(err))
def send(self, message, cb=None):
message[
"id"] = self.next_message_id # Add message id to allow response
self.next_message_id += 1
if cb: # Callback after client responded
self.waiting_cb[message["id"]] = cb
if self.sending:
return # Already sending
self.send_queue.append(message)
try:
while self.send_queue:
self.sending = True
message = self.send_queue.pop(0)
self.ws.send(json.dumps(message))
self.sending = False
except Exception, err:
self.log.debug("Websocket send error: %s" %
Debug.formatException(err))
def onUpdated(self, inner_path):
file_path = self.getPath(inner_path)
# Update Sql cache
if inner_path == "dbschema.json":
self.has_db = self.isFile("dbschema.json")
# Reopen DB to check changes
self.closeDb()
self.openDb()
elif not config.disable_db and inner_path.endswith(
".json") and self.has_db: # Load json file to db
if config.verbose:
self.log.debug("Loading json file to db: %s" % inner_path)
try:
self.getDb().loadJson(file_path)
except Exception, err:
self.log.error("Json %s load error: %s" %
(inner_path, Debug.formatException(err)))
self.closeDb()
def loadPlugins(self):
s = time.time()
for dir_name in sorted(os.listdir(self.plugin_path)):
dir_path = os.path.join(self.plugin_path, dir_name)
if dir_name.startswith("disabled"):
continue # Dont load if disabled
if not os.path.isdir(dir_path):
continue # Dont load if not dir
if dir_name.startswith("Debug") and not config.debug:
continue # Only load in debug mode if module name starts with Debug
self.log.debug("Loading plugin: %s" % dir_name)
try:
__import__(dir_name)
except Exception, err:
self.log.error("Plugin %s load error: %s" %
(dir_name, Debug.formatException(err)))
if dir_name not in self.plugin_names:
self.plugin_names.append(dir_name)
def resolveDomainDnschainInfo(self, domain):
try:
match = self.isDomain(domain)
sub_domain = match.group(1).strip(".")
top_domain = match.group(2)
if not sub_domain: sub_domain = "@"
address = None
with gevent.Timeout(5, Exception("Timeout: 5s")):
res = Http.get("https://dnschain.info/bit/d/%s" % re.sub("\.bit$", "", top_domain)).read()
data = json.loads(res)["value"]
for key, val in data["zeronet"].iteritems():
self.dns_cache[key+"."+top_domain] = [val, time.time()+60*60*5] # Cache for 5 hours
self.saveDnsCache()
return data["zeronet"].get(sub_domain)
# Not found
return address
except Exception, err:
log.debug("Dnschain.info %s resolve error: %s" % (domain, Debug.formatException(err)))
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = timeout = 5 + int(
file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 5sec + size in kb
timeout = timeout = 5 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request(
"update", {
"site":
self.address,
"inner_path":
inner_path,
"body":
body,
"peer":
(config.ip_external, config.fileserver_port)
})
if result:
break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
def handleStream(self, message, unpacker, buff, unpacker_bytes):
stream_bytes_left = message["stream_bytes"]
file = self.waiting_streams[message["to"]]
if "tell" in dir(unpacker):
unpacker_unprocessed_bytes = unpacker_bytes - unpacker.tell()
else:
unpacker_unprocessed_bytes = unpacker._fb_buf_n - unpacker._fb_buf_o
if unpacker_unprocessed_bytes: # Found stream bytes in unpacker
unpacker_stream_bytes = min(unpacker_unprocessed_bytes,
stream_bytes_left)
buff_stream_start = len(buff) - unpacker_unprocessed_bytes
file.write(buff[buff_stream_start:buff_stream_start +
unpacker_stream_bytes])
stream_bytes_left -= unpacker_stream_bytes
else:
unpacker_stream_bytes = 0
if config.debug_socket:
self.log(
"Starting stream %s: %s bytes (%s from unpacker, buff size: %s, unprocessed: %s)"
%
(message["to"], message["stream_bytes"], unpacker_stream_bytes,
len(buff), unpacker_unprocessed_bytes))
try:
while 1:
if stream_bytes_left <= 0:
break
stream_buff = self.sock.recv(min(64 * 1024, stream_bytes_left))
if not stream_buff:
break
buff_len = len(stream_buff)
stream_bytes_left -= buff_len
file.write(stream_buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
except Exception, err:
self.log("Stream read error: %s" % Debug.formatException(err))
def connectController(self):
if "socket_noproxy" in dir(
socket): # Socket proxy-patched, use non-proxy one
conn = socket.socket_noproxy(socket.AF_INET, socket.SOCK_STREAM)
else:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.log.info("Connecting to Tor Controller %s:%s" %
(self.ip, self.port))
try:
with self.lock:
conn.connect((self.ip, self.port))
# Auth cookie file
res_protocol = self.send("PROTOCOLINFO", conn)
cookie_match = re.search('COOKIEFILE="(.*?)"', res_protocol)
if cookie_match:
cookie_file = cookie_match.group(1).decode("string-escape")
auth_hex = binascii.b2a_hex(open(cookie_file, "rb").read())
res_auth = self.send("AUTHENTICATE %s" % auth_hex, conn)
elif config.tor_password:
res_auth = self.send(
'AUTHENTICATE "%s"' % config.tor_password, conn)
else:
res_auth = self.send("AUTHENTICATE", conn)
assert "250 OK" in res_auth, "Authenticate error %s" % res_auth
# Version 0.2.7.5 required because ADD_ONION support
res_version = self.send("GETINFO version", conn)
version = re.search('version=([0-9\.]+)', res_version).group(1)
assert float(
version.replace(".", "0", 2)
) >= 207.5, "Tor version >=0.2.7.5 required, found: %s" % version
self.setStatus(u"Connected (%s)" % res_auth)
self.conn = conn
except Exception, err:
self.conn = None
self.setStatus(u"Error (%s)" % str(err).decode("utf8", "ignore"))
self.log.warning(
u"Tor controller connect error: %s" %
Debug.formatException(str(err).decode("utf8", "ignore")))
self.enabled = False
def updateMergerSites(self):
global merger_db, merged_db, merged_to_merger, site_manager
s = time.time()
merger_db = {}
merged_db = {}
merged_to_merger = {}
site_manager = self
if not self.sites:
return
for site in self.sites.values():
# Update merged sites
try:
merged_type = site.content_manager.contents.get("content.json", {}).get("merged_type")
except Exception as err:
self.log.error("Error loading site %s: %s" % (site.address, Debug.formatException(err)))
continue
if merged_type:
merged_db[site.address] = merged_type
# Update merger sites
for permission in site.settings["permissions"]:
if not permission.startswith("Merger:"):
continue
if merged_type:
self.log.error(
"Removing permission %s from %s: Merger and merged at the same time." %
(permission, site.address)
)
site.settings["permissions"].remove(permission)
continue
merger_type = permission.replace("Merger:", "")
if site.address not in merger_db:
merger_db[site.address] = []
merger_db[site.address].append(merger_type)
site_manager.sites[site.address] = site
# Update merged to merger
if merged_type:
for merger_site in self.sites.values():
if "Merger:" + merged_type in merger_site.settings["permissions"]:
if site.address not in merged_to_merger:
merged_to_merger[site.address] = []
merged_to_merger[site.address].append(merger_site)
self.log.debug("Updated merger sites in %.3fs" % (time.time() - s))
def atomicWrite(dest, content, mode="w"):
try:
with open(dest + "-tmpnew", mode) as f:
f.write(content)
f.flush()
os.fsync(f.fileno())
if os.path.isfile(dest + "-tmpold"): # Previous incomplete write
os.rename(dest + "-tmpold", dest + "-tmpold-%s" % time.time())
os.rename(dest, dest + "-tmpold")
os.rename(dest + "-tmpnew", dest)
os.unlink(dest + "-tmpold")
return True
except Exception, err:
from Debug import Debug
logging.error("File %s write failed: %s, reverting..." %
(dest, Debug.formatException(err)))
if os.path.isfile(dest + "-tmpold") and not os.path.isfile(dest):
os.rename(dest + "-tmpold", dest)
return False
def siteSign(self,
address,
privatekey=None,
passphrase=False,
inner_path="content.json",
publish=False,
remove_missing_optional=False):
from Site import Site
from Site import SiteManager
from Debug import Debug
SiteManager.site_manager.load()
address = getPublicKey(address)
logging.info("Signing site: %s..." % address)
site = Site(address, allow_create=False)
if passphrase:
import getpass
passphrase = getpass.getpass(
"Type your passphrase (input hidden): ")
privatekey = hashlib.sha256(
passphrase if isinstance(passphrase, bytes) else passphrase.
encode("utf-8")).hexdigest()
elif not privatekey: # If no privatekey defined
from User import UserManager
user = UserManager.user_manager.get()
if user:
site_data = user.getSiteData(address)
privatekey = site_data.get("privatekey")
else:
privatekey = None
if not privatekey:
# Not found in users.json, ask from console
import getpass
privatekey = getpass.getpass("Private key (input hidden):")
try:
succ = site.content_manager.sign(
inner_path=inner_path,
privatekey=privatekey,
update_changed_files=True,
remove_missing_optional=remove_missing_optional)
except Exception, err:
logging.error("Sign error: %s" % Debug.formatException(err))
succ = False
def handleIncomingConnection(self, sock):
self.log("Incoming connection...")
if "TCP_NODELAY" in dir(socket):
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.type = "in"
if self.ip not in config.ip_local: # Clearnet: Check implicit SSL
try:
first_byte = sock.recv(1, gevent.socket.MSG_PEEK)
if first_byte == b"\x16":
self.log("Crypt in connection using implicit SSL")
self.sock = CryptConnection.manager.wrapSocket(
self.sock, "tls-rsa", True)
self.sock_wrapped = True
self.crypt = "tls-rsa"
except Exception as err:
self.log("Socket peek error: %s" % Debug.formatException(err))
self.messageLoop()
def connect(self, connection=None):
if self.connection:
self.log("Getting connection (Closing %s)..." % self.connection)
self.connection.close()
else:
self.log("Getting connection...")
if connection: # Connection specified
self.connection = connection
else: # Try to find from connection pool or create new connection
self.connection = None
try:
self.connection = self.site.connection_server.getConnection(self.ip, self.port)
except Exception, err:
self.onConnectionError()
self.log("Getting connection error: %s (connection_error: %s, hash_failed: %s)" %
(Debug.formatException(err), self.connection_error, self.hash_failed))
self.connection = None
def startTor(self):
if sys.platform.startswith("win"):
try:
if not os.path.isfile(self.tor_exe):
self.downloadTor()
self.log.info("Starting Tor client %s..." % self.tor_exe)
tor_dir = os.path.dirname(self.tor_exe)
self.tor_process = subprocess.Popen(r"%s -f torrc" % self.tor_exe, cwd=tor_dir, close_fds=True)
for wait in range(1,10): # Wait for startup
time.sleep(wait * 0.5)
self.enabled = True
if self.connect():
break
# Terminate on exit
atexit.register(self.stopTor)
except Exception, err:
self.log.error("Error starting Tor client: %s" % Debug.formatException(err))
self.enabled = False
def openArchive(self, inner_path):
archive_path = self.getPath(inner_path)
file_obj = None
if archive_path not in archive_cache:
if not os.path.isfile(archive_path):
result = self.site.needFile(inner_path, priority=10)
self.site.updateWebsocket(file_done=inner_path)
if not result:
raise Exception("Unable to download file")
file_obj = self.site.storage.openBigfile(inner_path)
if file_obj == False:
file_obj = None
try:
archive = openArchive(archive_path, file_obj=file_obj)
except Exception as err:
raise Exception("Unable to download file: %s" % Debug.formatException(err))
return archive
def messageLoop(self):
if not self.sock:
self.log("Socket error: No socket found")
return False
self.protocol = "v2"
self.updateName()
self.connected = True
buff_len = 0
self.unpacker = msgpack.fallback.Unpacker(
) # Due memory problems of C version
try:
while not self.closed:
buff = self.sock.recv(64 * 1024)
if not buff:
break # Connection closed
buff_len = len(buff)
# Statistics
self.last_recv_time = time.time()
self.incomplete_buff_recv += 1
self.bytes_recv += buff_len
self.server.bytes_recv += buff_len
if not self.unpacker:
self.unpacker = msgpack.fallback.Unpacker()
self.unpacker.feed(buff)
buff = None
for message in self.unpacker:
try:
self.incomplete_buff_recv = 0
if "stream_bytes" in message:
self.handleStream(message)
else:
self.handleMessage(message)
except TypeError:
raise Exception("Invalid message type: %s" %
type(message))
message = None
except Exception, err:
if not self.closed:
self.log("Socket error: %s" % Debug.formatException(err))
def connect(self, connection=None):
if self.reputation < -10:
self.reputation = -10
if self.reputation > 10:
self.reputation = 10
if self.connection:
self.log("Getting connection (Closing %s)..." % self.connection)
self.connection.close("Connection change")
else:
self.log("Getting connection (reputation: %s)..." %
self.reputation)
if connection: # Connection specified
self.log("Assigning connection %s" % connection)
self.connection = connection
self.connection.sites += 1
else: # Try to find from connection pool or create new connection
self.connection = None
try:
if self.connection_server:
connection_server = self.connection_server
elif self.site:
connection_server = self.site.connection_server
else:
import main
connection_server = main.file_server
self.connection = connection_server.getConnection(
self.ip,
self.port,
site=self.site,
is_tracker_connection=self.is_tracker_connection)
self.reputation += 1
self.connection.sites += 1
except Exception as err:
self.onConnectionError("Getting connection error")
self.log(
"Getting connection error: %s (connection_error: %s, hash_failed: %s)"
% (Debug.formatException(err), self.connection_error,
self.hash_failed))
self.connection = None
return self.connection
def request(self, cmd, params={}, stream_to=None):
if not self.connection or self.connection.closed:
self.connect()
if not self.connection:
self.onConnectionError("Reconnect error")
return None # Connection failed
self.log("Send request: %s %s %s %s" %
(params.get("site", ""), cmd, params.get(
"inner_path", ""), params.get("location", "")))
for retry in range(1, 2): # Retry 1 times
try:
if not self.connection:
raise Exception("No connection found")
res = self.connection.request(cmd, params, stream_to)
if not res:
raise Exception("Send error")
if "error" in res:
self.log("%s error: %s" % (cmd, res["error"]))
self.onConnectionError("Response error")
break
else: # Successful request, reset connection error num
self.connection_error = 0
self.time_response = time.time()
if res:
return res
else:
raise Exception("Invalid response: %s" % res)
except Exception as err:
if type(err).__name__ == "Notify": # Greenlet killed by worker
self.log("Peer worker got killed: %s, aborting cmd: %s" %
(err.message, cmd))
break
else:
self.onConnectionError("Request error")
self.log(
"%s (connection_error: %s, hash_failed: %s, retry: %s)"
% (Debug.formatException(err), self.connection_error,
self.hash_failed, retry))
time.sleep(1 * retry)
self.connect()
return None # Failed after 4 retry
def testOpenportPortchecker(self, port=None):
self.log.info("Checking port %s using portchecker.co..." % port)
if config.i2p == "enable" and config.tor == "enable":
please_not_test = True
else:
please_not_test = False
try:
data = urllib2.urlopen("https://portchecker.co/check",
"port=%s" % port,
timeout=20.0).read()
message = re.match('.*<div id="results-wrapper">(.*?)</div>', data,
re.DOTALL).group(1)
message = re.sub("<.*?>", "",
message.replace("<br>", " ").replace(
" ", " ").strip()) # Strip http tags
except Exception, err:
message = "Error: %s" % Debug.formatException(err)
data = ""
def sendCmd(self, cmd, params={}):
if not self.socket: self.connect()
if cmd != "ping" and self.last_response and time.time(
) - self.last_response > 20 * 60: # If last response if older than 20 minute, ping first to see if still alive
if not self.ping(): return None
for retry in range(1, 3): # Retry 3 times
if config.debug_socket:
self.log.debug("sendCmd: %s %s" %
(cmd, params.get("inner_path")))
try:
self.socket.send(
msgpack.packb({
"cmd": cmd,
"params": params
},
use_bin_type=True))
if config.debug_socket:
self.log.debug("Sent command: %s" % cmd)
response = msgpack.unpackb(self.socket.recv())
if config.debug_socket:
self.log.debug("Got response to: %s" % cmd)
if "error" in response:
self.log.debug("%s error: %s" % (cmd, response["error"]))
self.onConnectionError()
else: # Successful request, reset connection error num
self.connection_error = 0
self.last_response = time.time()
return response
except Exception, err:
self.onConnectionError()
self.log.debug(
"%s (connection_error: %s, hash_failed: %s, retry: %s)" %
(Debug.formatException(err), self.connection_error,
self.hash_failed, retry))
time.sleep(1 * retry)
self.connect()
if type(
err
).__name__ == "Notify" and err.message == "Worker stopped": # Greenlet kill by worker
self.log.debug("Peer worker got killed, aborting cmd: %s" %
cmd)
break
def connect(self):
if not self.enabled:
return False
self.site_onions = {}
self.privatekeys = {}
if "socket_noproxy" in dir(
socket): # Socket proxy-patched, use non-proxy one
conn = socket.socket_noproxy(socket.AF_INET, socket.SOCK_STREAM)
else:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.log.debug("Connecting to %s:%s" % (self.ip, self.port))
try:
with self.lock:
conn.connect((self.ip, self.port))
res_protocol = self.send("PROTOCOLINFO", conn)
version = re.search('Tor="([0-9\.]+)', res_protocol).group(1)
# Version 0.2.7.5 required because ADD_ONION support
assert float(
version.replace(".", "0", 2)
) >= 207.5, "Tor version >=0.2.7.5 required, found: %s" % version
# Auth cookie file
cookie_match = re.search('COOKIEFILE="(.*?)"', res_protocol)
if cookie_match:
cookie_file = cookie_match.group(1)
auth_hex = binascii.b2a_hex(open(cookie_file, "rb").read())
res_auth = self.send("AUTHENTICATE %s" % auth_hex, conn)
else:
res_auth = self.send("AUTHENTICATE", conn)
assert "250 OK" in res_auth, "Authenticate error %s" % res_auth
self.status = u"Connected (%s)" % res_auth
self.conn = conn
except Exception, err:
self.conn = None
self.status = u"Error (%s)" % err
self.log.error("Tor controller connect error: %s" %
Debug.formatException(err))
self.enabled = False
def actionSiteSign(self, to, privatekey=None, inner_path="content.json", response_ok=True, update_changed_files=False, remove_missing_optional=False):
self.log.debug("Signing: %s" % inner_path)
site = self.site
extend = {} # Extended info for signing
# Change to the file's content.json
file_info = site.content_manager.getFileInfo(inner_path)
if not inner_path.endswith("content.json"):
if not file_info:
raise Exception("Invalid content.json file: %s" % inner_path)
inner_path = file_info["content_inner_path"]
# Add certificate to user files
if file_info and "cert_signers" in file_info and privatekey is None:
cert = self.user.getCert(self.site.address)
extend["cert_auth_type"] = cert["auth_type"]
extend["cert_user_id"] = self.user.getCertUserId(site.address)
extend["cert_sign"] = cert["cert_sign"]
if (
not site.settings["own"] and
self.user.getAuthAddress(self.site.address) not in self.site.content_manager.getValidSigners(inner_path)
):
self.log.error("SiteSign error: you don't own this site & site owner doesn't allow you to do so.")
return self.response(to, {"error": "Forbidden, you can only modify your own sites"})
if privatekey == "stored": # Get privatekey from sites.json
privatekey = self.user.getSiteData(self.site.address).get("privatekey")
if not privatekey: # Get privatekey from users.json auth_address
privatekey = self.user.getAuthPrivatekey(self.site.address)
# Signing
# Reload content.json, ignore errors to make it up-to-date
site.content_manager.loadContent(inner_path, add_bad_files=False, force=True)
# Sign using private key sent by user
try:
signed = site.content_manager.sign(inner_path, privatekey, extend=extend, update_changed_files=update_changed_files, remove_missing_optional=remove_missing_optional)
except Exception, err:
self.cmd("notification", ["error", _["Content signing failed"] + "<br><small>%s</small>" % err])
self.response(to, {"error": "Site sign failed: %s" % err})
self.log.error("Site sign failed: %s: %s" % (inner_path, Debug.formatException(err)))
return
def includeUpdateAll(self, update_site_dbs=True):
s = time.time()
new_include_filters = collections.defaultdict(set)
# Load all include files data into a merged set
for include_path in self.file_content["includes"]:
address, inner_path = include_path.split("/", 1)
try:
content = self.site_manager.get(address).storage.loadJson(
inner_path)
except Exception as err:
self.log.warning("Error loading include %s: %s" %
(include_path, Debug.formatException(err)))
continue
for key, val in content.items():
if type(val) is not dict:
continue
new_include_filters[key].update(val.keys())
mutes_added = new_include_filters["mutes"].difference(
self.include_filters["mutes"])
mutes_removed = self.include_filters["mutes"].difference(
new_include_filters["mutes"])
self.include_filters = new_include_filters
if update_site_dbs:
for auth_address in mutes_added:
self.changeDbs(auth_address, "remove")
for auth_address in mutes_removed:
if not self.isMuted(auth_address):
self.changeDbs(auth_address, "load")
num_mutes = len(self.include_filters["mutes"])
num_siteblocks = len(self.include_filters["siteblocks"])
self.log.debug(
"Loaded %s mutes, %s blocked sites from %s includes in %.3fs" %
(num_mutes, num_siteblocks, len(
self.file_content["includes"]), time.time() - s))
def zmqServer(self):
if config.disable_zeromq:
self.log.debug("ZeroMQ disabled by config")
return False
self.log.debug("Starting ZeroMQ on: tcp://127.0.0.1:%s..." %
self.zmq_port)
try:
import zmq.green as zmq
context = zmq.Context()
self.zmq_sock = context.socket(zmq.REP)
self.zmq_sock.bind("tcp://127.0.0.1:%s" % self.zmq_port)
self.zmq_sock.hwm = 1
self.zmq_sock.setsockopt(zmq.RCVTIMEO,
5000) # Wait for data receive
self.zmq_sock.setsockopt(zmq.SNDTIMEO, 50000) # Wait for data send
self.zmq_running = True
except Exception, err:
self.log.debug("ZeroMQ start error: %s" %
Debug.formatException(err))
return False
def loadContent(self,
content_inner_path="content.json",
add_bad_files=True,
delete_removed_files=True,
load_includes=True):
content_inner_path = content_inner_path.strip(
"/") # Remove / from begning
old_content = self.contents.get(content_inner_path)
content_path = self.site.storage.getPath(content_inner_path)
content_dir = helper.getDirname(
self.site.storage.getPath(content_inner_path))
content_inner_dir = helper.getDirname(content_inner_path)
if os.path.isfile(content_path):
try:
new_content = json.load(open(content_path))
except Exception, err:
self.log.error("%s load error: %s" %
(content_path, Debug.formatException(err)))
return [], []
def onUpdated(self, inner_path, file=None):
# Update Sql cache
if inner_path == "dbschema.json":
self.has_db = self.isFile("dbschema.json")
# Reopen DB to check changes
if self.has_db:
self.closeDb()
self.getDb()
elif not config.disable_db and (
inner_path.endswith(".json") or inner_path.endswith(".json.gz")
) and self.has_db: # Load json file to db
if config.verbose:
self.log.debug("Loading json file to db: %s (file: %s)" %
(inner_path, file))
try:
self.updateDbFile(inner_path, file)
except Exception, err:
self.log.error("Json %s load error: %s" %
(inner_path, Debug.formatException(err)))
self.closeDb()
def openport(self, port=None, check=True):
if not port:
port = self.port
if self.port_opened:
return True # Port already opened
if check: # Check first if its already opened
time.sleep(1) # Wait for port open
if self.testOpenport(port, use_alternative=False)["result"] is True:
return True # Port already opened
if config.tor == "always": # Port opening won't work in Tor mode
return False
self.log.info("Trying to open port using UpnpPunch...")
try:
upnp_punch = UpnpPunch.open_port(self.port, 'ZeroNet')
upnp_punch = True
except Exception, err:
self.log.error("UpnpPunch run error: %s" % Debug.formatException(err))
upnp_punch = False
def actionSiteMedia(self, path, **kwargs):
if ".zip/" in path or ".tar.gz/" in path:
file_obj = None
path_parts = self.parsePath(path)
file_path = "%s/%s/%s" % (config.data_dir, path_parts["address"], path_parts["inner_path"])
match = re.match("^(.*\.(?:tar.gz|tar.bz2|zip))/(.*)", file_path)
archive_path, path_within = match.groups()
if archive_path not in archive_cache:
site = self.server.site_manager.get(path_parts["address"])
if not site:
return self.actionSiteAddPrompt(path)
archive_inner_path = site.storage.getInnerPath(archive_path)
if not os.path.isfile(archive_path):
# Wait until file downloads
result = site.needFile(archive_inner_path, priority=10)
# Send virutal file path download finished event to remove loading screen
site.updateWebsocket(file_done=archive_inner_path)
if not result:
return self.error404(archive_inner_path)
file_obj = site.storage.openBigfile(archive_inner_path)
if file_obj == False:
file_obj = None
header_allow_ajax = False
if self.get.get("ajax_key"):
requester_site = self.server.site_manager.get(path_parts["request_address"])
if self.get["ajax_key"] == requester_site.settings["ajax_key"]:
header_allow_ajax = True
else:
return self.error403("Invalid ajax_key")
try:
file = openArchiveFile(archive_path, path_within, file_obj=file_obj)
content_type = self.getContentType(file_path)
self.sendHeader(200, content_type=content_type, noscript=kwargs.get("header_noscript", False), allow_ajax=header_allow_ajax)
return self.streamFile(file)
except Exception as err:
self.log.debug("Error opening archive file: %s" % Debug.formatException(err))
return self.error404(path)
return super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs)
def getConnection(self, ip=None, port=None, peer_id=None):
if peer_id and peer_id in self.peer_ids: # Find connection by peer id
connection = self.peer_ids.get(peer_id)
if not connection.connected:
succ = connection.event_connected.get() # Wait for connection
if not succ: raise Exception("Connection event return error")
return connection
# Find connection by ip
if ip in self.ips:
connection = self.ips[ip]
if not connection.connected:
succ = connection.event_connected.get() # Wait for connection
if not succ: raise Exception("Connection event return error")
return connection
# Recover from connection pool
for connection in self.connections:
if connection.ip == ip:
if not connection.connected:
succ = connection.event_connected.get(
) # Wait for connection
if not succ:
raise Exception("Connection event return error")
return connection
# No connection found
if port == 0:
raise Exception("This peer is not connectable")
try:
connection = Connection(self, ip, port)
self.ips[ip] = connection
self.connections.append(connection)
succ = connection.connect()
if not succ:
connection.close()
raise Exception("Connection event return error")
except Exception, err:
self.log.debug("%s Connect error: %s" %
(ip, Debug.formatException(err)))
connection.close()
raise err
def updatePiecefields(self, force=False):
if self.connection and self.connection.handshake.get("rev", 0) < 2190:
return False # Not supported
# Don't update piecefield again in 1 min
if self.time_piecefields_updated and time.time() - self.time_piecefields_updated < 60 and not force:
return False
self.time_piecefields_updated = time.time()
res = self.request("getPiecefields", {"site": self.site.address})
if not res or "error" in res:
return False
self.piecefields = collections.defaultdict(BigfilePiecefieldPacked)
try:
for sha512, piecefield_packed in res["piecefields_packed"].iteritems():
self.piecefields[sha512].unpack(piecefield_packed)
except Exception as err:
self.log("Invalid updatePiecefields response: %s" % Debug.formatException(err))
return self.piecefields
def close(self):
if self.closed: return False # Already closed
self.closed = True
self.connected = False
self.event_connected.set(False)
if config.debug_socket: self.log("Closing connection, waiting_requests: %s, buff: %s..." % (len(self.waiting_requests), self.incomplete_buff_recv))
for request in self.waiting_requests.values(): # Mark pending requests failed
request.set(False)
self.waiting_requests = {}
self.server.removeConnection(self) # Remove connection from server registry
try:
if self.forward_thread:
self.forward_thread.kill(exception=Debug.Notify("Closing connection"))
if self.zmq_sock:
self.zmq_sock.close()
if self.sock:
self.sock.shutdown(gevent.socket.SHUT_WR)
self.sock.close()
except Exception, err:
if config.debug_socket: self.log("Close error: %s" % Debug.formatException(err))
