class Site(object):
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.time_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0,
10) # Last announced tracker id
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {
} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(
self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(
sys.modules["main"]
): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
self.settings = {
"own": False,
"serving": True,
"permissions": [],
"added": int(time.time())
} # Default
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings[
"permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
s = time.time()
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite(
"%s/sites.json" % config.data_dir,
json.dumps(sites_settings, indent=2, sort_keys=True))
self.log.debug("Saved settings in %.2fs" % (time.time() - s))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self,
inner_path,
download_files=True,
peer=None,
check_modifications=False,
diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" %
(inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(
lambda file_name: self.checkModifications(0),
"check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(
inner_path, load_includes=False)
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[
inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
new_file = Diff.patch(
self.storage.open(file_inner_path, "rb"),
diff_actions)
new_file.seek(0)
diff_success = self.content_manager.verifyFile(
file_inner_path, new_file)
if diff_success:
self.log.debug("Patched successfully: %s" %
file_inner_path)
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
self.onFileDone(file_inner_path)
except Exception, err:
self.log.debug("Failed to patch %s: %s" %
(file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[
inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent,
file_inner_path,
download_files=download_files,
peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." %
(inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." %
(inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.2fs" %
(inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0, 10) # Last announced tracker id
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = {"own": False, "serving": True, "permissions": permissions} # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite("%s/sites.json" % config.data_dir, json.dumps(sites_settings, indent=2, sort_keys=True))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files:
return False
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False, blind_includes=False):
self.log.debug(
"Start downloading, bad_files: %s, check_size: %s, blind_includes: %s" %
(self.bad_files, check_size, blind_includes)
)
gevent.spawn(self.announce)
if check_size: # Check the size first
valid = self.downloadContent("content.json", download_files=False) # Just download content.json files
if not valid:
return False # Cant download content.jsons or size is not fits
# Download everything
valid = self.downloadContent("content.json", check_modifications=blind_includes)
self.retryBadFiles()
return valid
# Update worker, try to find client that supports listModifications command
def updater(self, peers_try, queried, since):
while 1:
if not peers_try or len(queried) >= 3: # Stop after 3 successful query
break
peer = peers_try.pop(0)
if not peer.connection and len(queried) < 2:
peer.connect() # Only open new connection if less than 2 queried already
if not peer.connection or peer.connection.handshake.get("rev", 0) < 126:
continue # Not compatible
res = peer.listModified(since)
if not res or "modified_files" not in res:
continue # Failed query
queried.append(peer)
for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we
content = self.content_manager.contents.get(inner_path)
if (not content or modified > content["modified"]) and inner_path not in self.bad_files:
self.log.debug("New modified file from %s: %s" % (peer, inner_path))
# We dont have this file or we have older
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0) + 1 # Mark as bad file
gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files
# Check modified content.json files from peers and add modified files to bad_files
# Return: Successfully queried peers [Peer, Peer...]
def checkModifications(self, since=None):
peers_try = [] # Try these peers
queried = [] # Successfully queried from these peers
peers = self.peers.values()
random.shuffle(peers)
for peer in peers: # Try to find connected good peers, but we must have at least 5 peers
if peer.findConnection() and peer.connection.handshake.get("rev", 0) > 125: # Add to the beginning if rev125
peers_try.insert(0, peer)
elif len(peers_try) < 5: # Backup peers, add to end of the try list
peers_try.append(peer)
if since is None: # No since definied, download from last modification time-1day
since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24
self.log.debug("Try to get listModifications from peers: %s since: %s" % (peers_try, since))
updaters = []
for i in range(3):
updaters.append(gevent.spawn(self.updater, peers_try, queried, since))
gevent.joinall(updaters, timeout=10) # Wait 10 sec to workers done query modifications
time.sleep(0.1)
self.log.debug("Queried listModifications from: %s" % queried)
return queried
# Update content.json from peers and download changed files
# Return: None
@util.Noparallel()
def update(self, announce=False):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None # Reset content updated time
self.updateWebsocket(updating=True)
if announce:
self.announce()
queried = self.checkModifications()
if not queried: # Not found any client that supports listModifications
self.log.debug("Fallback to old-style update")
self.redownloadContents()
if not self.settings["own"]:
self.storage.checkFiles(quick_check=True) # Quick check files based on file size
changed, deleted = self.content_manager.loadContent("content.json")
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize() # Update site size
self.updateWebsocket(updated=True)
# Update site by redownload all content.json
def redownloadContents(self):
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
gevent.joinall(content_threads)
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = timeout = 5 + int(file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 5sec + size in kb
timeout = timeout = 5 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (config.ip_external, config.fileserver_port)
})
if result:
break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = { "own": False, "serving": True, "permissions": permissions } # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size*1.2 < size_limit*1024*1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path, load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time()-s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files: return False
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
gevent.spawn(self.announce)
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid: return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update worker, try to find client that supports listModifications command
def updater(self, peers_try, queried):
since = self.settings.get("modified", 60*60*24)-60*60*24 # Get modified since last update - 1day
while 1:
if not peers_try or len(queried) >= 3: # Stop after 3 successful query
break
peer = peers_try.pop(0)
if not peer.connection and len(queried) < 2: peer.connect() # Only open new connection if less than 2 queried already
if not peer.connection or peer.connection.handshake.get("rev",0) < 126: continue # Not compatible
res = peer.listModified(since)
if not res or not "modified_files" in res: continue # Failed query
queried.append(peer)
for inner_path, modified in res["modified_files"].iteritems(): # Check if the peer has newer files than we
content = self.content_manager.contents.get(inner_path)
if not content or modified > content["modified"]: # We dont have this file or we have older
self.bad_files[inner_path] = self.bad_files.get(inner_path, 0)+1 # Mark as bad file
gevent.spawn(self.downloadContent, inner_path) # Download the content.json + the changed files
# Update content.json from peers and download changed files
# Return: None
@util.Noparallel()
def update(self, announce=False):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None # Reset content updated time
self.updateWebsocket(updating=True)
if announce: self.announce()
peers_try = [] # Try these peers
queried = [] # Successfully queried from these peers
peers = self.peers.values()
random.shuffle(peers)
for peer in peers: # Try to find connected good peers, but we must have at least 5 peers
if peer.findConnection() and peer.connection.handshake.get("rev",0) > 125: # Add to the beginning if rev125
peers_try.insert(0, peer)
elif len(peers_try) < 5: # Backup peers, add to end of the try list
peers_try.append(peer)
self.log.debug("Try to get listModifications from peers: %s" % peers_try)
updaters = []
for i in range(3):
updaters.append(gevent.spawn(self.updater, peers_try, queried))
gevent.joinall(updaters, timeout=5) # Wait 5 sec to workers
time.sleep(0.1)
self.log.debug("Queried listModifications from: %s" % queried)
if not queried: # Not found any client that supports listModifications
self.log.debug("Fallback to old-style update")
self.redownloadContents()
if not self.settings["own"]: self.storage.checkFiles(quick_check=True) # Quick check files based on file size
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = self.bad_files.get(changed_file, 0)+1
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize() # Update site size
self.updateWebsocket(updated=True)
# Update site by redownload all content.json
def redownloadContents(self):
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
gevent.joinall(content_threads)
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
timeout = timeout = 5+int(file_size/1024)+peer.connection.last_ping_delay # Timeout: 5sec + size in kb + last_ping
else:
timeout = timeout = 5+int(file_size/1024) # Timeout: 5sec + size in kb
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}: peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site(object):
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.time_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0, 10) # Last announced tracker id
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
self.settings = {"own": False, "serving": True, "permissions": [], "added": int(time.time())} # Default
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite("%s/sites.json" % config.data_dir, json.dumps(sites_settings, indent=2, sort_keys=True))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
new_file.seek(0)
diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
if diff_success:
self.log.debug("Patched successfully: %s" % file_inner_path)
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
self.onFileDone(file_inner_path)
except Exception, err:
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = { "own": False, "serving": True, "permissions": permissions } # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size*1.2 < size_limit*1024*1024:
return size_limit
return 999999
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path, load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time()-s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files: return False
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid: return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update content.json from peers and download changed files
@util.Noparallel()
def update(self):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
gevent.joinall(content_threads)
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = self.bad_files.get(changed_file, 0)+1
if not self.settings["own"]: self.storage.checkFiles(quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize() # Update site size
return changed
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
timeout = 5+int(self.storage.getSize(inner_path)/1024) # Timeout: 5sec + size in kb
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": self.storage.open(inner_path).read(),
"peer": (config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site(object):
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.time_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0,
10) # Last announced tracker id
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {
} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(
self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(
sys.modules["main"]
): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
self.settings = {
"own": False,
"serving": True,
"permissions": [],
"added": int(time.time())
} # Default
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings[
"permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite(
"%s/sites.json" % config.data_dir,
json.dumps(sites_settings, indent=2, sort_keys=True))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self,
inner_path,
download_files=True,
peer=None,
check_modifications=False):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" %
(inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(
lambda file_name: self.checkModifications(0),
"check_modifications")
return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(
inner_path, load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[
inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent,
file_inner_path,
download_files=download_files,
peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." %
(inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
self.log.debug("%s: Downloading %s files, changed: %s..." %
(inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: DownloadContent ended in %.2fs" %
(inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files:
return False
return [
bad_file for bad_file, retry in self.bad_files.iteritems()
if retry < 3
]
# Retry download bad files
def retryBadFiles(self, force=False):
for bad_file, tries in self.bad_files.items():
if force or random.randint(
0, min(20, tries)
) == 0: # Larger number tries = less likely to check every 15min
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False, blind_includes=False):
self.log.debug(
"Start downloading, bad_files: %s, check_size: %s, blind_includes: %s"
% (self.bad_files, check_size, blind_includes))
gevent.spawn(self.announce)
if check_size: # Check the size first
valid = self.downloadContent(
"content.json",
download_files=False) # Just download content.json files
if not valid:
return False # Cant download content.jsons or size is not fits
# Download everything
valid = self.downloadContent("content.json",
check_modifications=blind_includes)
self.retryBadFiles(force=True)
return valid
# Update worker, try to find client that supports listModifications command
def updater(self, peers_try, queried, since):
while 1:
if not peers_try or len(
queried) >= 3: # Stop after 3 successful query
break
peer = peers_try.pop(0)
if not peer.connection and len(queried) < 2:
peer.connect(
) # Only open new connection if less than 2 queried already
if not peer.connection or peer.connection.handshake.get("rev",
0) < 126:
continue # Not compatible
res = peer.listModified(since)
if not res or "modified_files" not in res:
continue # Failed query
queried.append(peer)
for inner_path, modified in res["modified_files"].iteritems(
): # Check if the peer has newer files than we
content = self.content_manager.contents.get(inner_path)
if (not content or modified > content["modified"]
) and inner_path not in self.bad_files:
self.log.debug("New modified file from %s: %s" %
(peer, inner_path))
# We dont have this file or we have older
self.bad_files[inner_path] = self.bad_files.get(
inner_path, 0) + 1 # Mark as bad file
gevent.spawn(
self.downloadContent, inner_path
) # Download the content.json + the changed files
# Check modified content.json files from peers and add modified files to bad_files
# Return: Successfully queried peers [Peer, Peer...]
def checkModifications(self, since=None):
peers_try = [] # Try these peers
queried = [] # Successfully queried from these peers
# Wait for peers
if not self.peers:
self.announce()
for wait in range(10):
time.sleep(5 + wait)
self.log.debug("Waiting for peers...")
if self.peers:
break
peers = self.peers.values()
random.shuffle(peers)
for peer in peers: # Try to find connected good peers, but we must have at least 5 peers
if peer.findConnection() and peer.connection.handshake.get(
"rev", 0) > 125: # Add to the beginning if rev125
peers_try.insert(0, peer)
elif len(
peers_try) < 5: # Backup peers, add to end of the try list
peers_try.append(peer)
if since is None: # No since defined, download from last modification time-1day
since = self.settings.get("modified", 60 * 60 * 24) - 60 * 60 * 24
self.log.debug(
"Try to get listModifications from peers: %s since: %s" %
(peers_try, since))
updaters = []
for i in range(3):
updaters.append(
gevent.spawn(self.updater, peers_try, queried, since))
gevent.joinall(
updaters,
timeout=10) # Wait 10 sec to workers done query modifications
if not queried:
gevent.joinall(
updaters,
timeout=10) # Wait another 10 sec if none of updaters finished
time.sleep(0.1)
self.log.debug("Queried listModifications from: %s" % queried)
return queried
# Update content.json from peers and download changed files
# Return: None
@util.Noparallel()
def update(self, announce=False):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None # Reset content updated time
self.updateWebsocket(updating=True)
if announce:
self.announce()
queried = self.checkModifications()
if not queried: # Not found any client that supports listModifications
self.log.debug("Fallback to old-style update")
self.redownloadContents()
self.storage.checkFiles(
quick_check=True
) # Quick check and mark bad files based on file size
changed, deleted = self.content_manager.loadContent("content.json")
if self.bad_files:
self.log.debug("Bad files: %s" % self.bad_files)
self.download()
self.settings["size"] = self.content_manager.getTotalSize(
) # Update site size
self.updateWebsocket(updated=True)
# Update site by redownload all content.json
def redownloadContents(self):
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(
self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." %
len(content_threads))
gevent.joinall(content_threads)
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
file_size = self.storage.getSize(inner_path)
body = self.storage.read(inner_path)
tor_manager = self.connection_server.tor_manager
if tor_manager.enabled and tor_manager.start_onions:
my_ip = tor_manager.getOnion(self.address)
if my_ip:
my_ip += ".onion"
my_port = config.fileserver_port
else:
my_ip = config.ip_external
if self.connection_server.port_opened:
my_port = config.fileserver_port
else:
my_port = 0
while 1:
if not peers or len(published) >= limit:
if event_done:
event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
if peer.connection and peer.connection.last_ping_delay: # Peer connected
# Timeout: 5sec + size in kb + last_ping
timeout = timeout = 5 + int(
file_size / 1024) + peer.connection.last_ping_delay
else: # Peer not connected
# Timeout: 5sec + size in kb
timeout = timeout = 5 + int(file_size / 1024)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request(
"update", {
"site": self.address,
"inner_path": inner_path,
"body": body,
"peer": (my_ip, my_port)
})
if result:
break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
if result == {"exception": "Timeout"}:
peer.onConnectionError()
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site(object):
def __init__(self, address, allow_create=True, settings=None):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_hash = hashlib.sha256(self.address).digest()
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peers_recent = collections.deque(maxlen=100)
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.websockets = [] # Active site websocket connections
self.connection_server = None
self.loadSettings(settings) # Load settings from sites.json
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.content_manager = ContentManager(self)
self.content_manager.loadContents() # Load content.json files
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.log.debug("Creating connection server") # remove
self.connection_server = FileServer()
self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
if not self.settings.get("ajax_key"): # To auth websocket permissions
self.settings["ajax_key"] = CryptHash.random()
self.log.debug("New ajax key: %s" % self.settings["ajax_key"])
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self, settings=None):
if not settings:
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
if settings:
self.settings = settings
if "cache" not in settings:
settings["cache"] = {}
if "size_files_optional" not in settings:
settings["size_optional"] = 0
if "optional_downloaded" not in settings:
settings["optional_downloaded"] = 0
if "downloaded" not in settings:
settings["downloaded"] = settings.get("added")
self.bad_files = settings["cache"].get("bad_files", {})
settings["cache"]["bad_files"] = {}
# Give it minimum 10 tries after restart
for inner_path in self.bad_files:
self.bad_files[inner_path] = min(self.bad_files[inner_path], 20)
else:
self.settings = {
"own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0,
"added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0
} # Default
if config.download_optional == "auto":
self.settings["autodownloadoptional"] = True
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
if not SiteManager.site_manager.sites:
SiteManager.site_manager.sites = {}
if not SiteManager.site_manager.sites.get(self.address):
SiteManager.site_manager.sites[self.address] = self
SiteManager.site_manager.load(False)
SiteManager.site_manager.save()
def getSettingsCache(self):
back = {}
back["bad_files"] = self.bad_files
back["hashfield"] = self.content_manager.hashfield.tostring().encode("base64")
return back
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", int(config.size_limit))
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
if not inner_path.endswith("content.json"):
return False
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
if inner_path == "content.json":
self.saveSettings()
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
s = time.time()
new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
new_file.seek(0)
time_diff = time.time() - s
s = time.time()
diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
time_verify = time.time() - s
if diff_success:
s = time.time()
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
time_write = time.time() - s
s = time.time()
self.onFileDone(file_inner_path)
time_on_done = time.time() - s
self.log.debug(
"Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" %
(file_inner_path, time_diff, time_verify, time_write, time_on_done)
)
except Exception, err:
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
if file_inner_path not in changed and not self.bad_files.get(file_inner_path):
continue
if not self.isDownloadable(file_inner_path):
continue
# Start download and dont wait for finish, return the event
res = self.pooledNeedFile(
file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer
)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.3fs" % (inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {
} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(
self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = {
"own": False,
"serving": True,
"permissions": permissions
} # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json",
"w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path,
load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent,
file_inner_path,
download_files=download_files,
peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." %
(inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." %
(inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" %
(inner_path, time.time() - s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files: return False
return [
bad_file for bad_file, retry in self.bad_files.iteritems()
if retry < 3
]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid:
return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update content.json from peers and download changed files
@util.Noparallel()
def update(self):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(
self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." %
len(content_threads))
gevent.joinall(content_threads)
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = self.bad_files.get(
changed_file, 0) + 1
if not self.settings["own"]:
self.storage.checkFiles(
quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize(
) # Update site size
return changed
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
timeout = 5 + int(self.storage.getSize(inner_path) /
1024) # Timeout: 5sec + size in kb
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request(
"update", {
"site":
self.address,
"inner_path":
inner_path,
"body":
self.storage.open(inner_path).read(),
"peer":
(config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.directory = "data/%s" % self.address # Site data diretory
self.log = logging.getLogger("Site:%s" % self.address_short)
if not os.path.isdir(self.directory):
if allow_create:
os.mkdir(self.directory) # Create directory if not found
else:
raise Exception("Directory not exists: %s" % self.directory)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {} # SHA512 check failed files, need to redownload
self.content_updated = None # Content.js update time
self.last_downloads = [] # Files downloaded in run of self.download()
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = {
"own": False,
"serving": True,
"permissions": permissions
} # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json",
"w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Sercurity check and return path of site's file
def getPath(self, inner_path):
inner_path = inner_path.replace("\\", "/") # Windows separator fix
inner_path = re.sub(
"^%s/" % re.escape(self.directory), "",
inner_path) # Remove site directory if begins with it
file_path = self.directory + "/" + inner_path
allowed_dir = os.path.abspath(
self.directory) # Only files within this directory allowed
if ".." in file_path or not os.path.dirname(
os.path.abspath(file_path)).startswith(allowed_dir):
raise Exception("File not allowed: %s" % file_path)
return file_path
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
self.last_downloads.append(inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path,
load_includes=False)
# Start download files
evts = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
self.last_downloads.append(file_inner_path)
evts.append(res) # Append evt
# Wait for includes download
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
self.downloadContent(file_inner_path,
download_files=download_files,
peer=peer)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files..." % (inner_path, len(evts)))
gevent.joinall(evts)
self.log.debug("%s: All file downloaded in %.2fs" %
(inner_path, time.time() - s))
return True
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
self.last_downloads = []
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid:
return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update content.json from peers and download changed files
@util.Noparallel()
def update(self):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None
# Download all content.json again
for inner_path in self.content_manager.contents.keys():
self.needFile(inner_path, update=True)
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = True
if not self.settings["own"]:
self.checkFiles(
quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize(
) # Update site size
return changed
def publisher(self, inner_path, peers, published, limit):
timeout = 5 + int(os.path.getsize(self.getPath(inner_path)) /
1024) # Timeout: 5sec + size in kb
while 1:
if not peers or len(published) >= limit:
break # All peers done, or published engouht
peer = peers.pop(0)
result = {"exception": "Timeout"}
try:
with gevent.Timeout(timeout, False):
result = peer.sendCmd(
"update", {
"site": self.address,
"inner_path": inner_path,
"body": open(self.getPath(inner_path),
"rb").read(),
"peer":
(config.ip_external, config.fileserver_port)
})
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
self.log.info("[ERROR] %s: %s" % (peer.key, result))