class Site(object): def __init__(self, address, allow_create=True): self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging self.log = logging.getLogger("Site:%s" % self.address_short) self.addEventListeners() self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) self.time_announce = 0 # Last announce time to tracker self.last_tracker_id = random.randint(0, 10) # Last announced tracker id self.worker_manager = WorkerManager(self) # Handle site download from other peers self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files self.loadSettings() # Load settings from sites.json self.content_manager = ContentManager(self) # Load contents self.connection_server = None if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible self.connection_server = sys.modules["main"].file_server else: self.connection_server = None if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed) self.settings["auth_key"] = CryptHash.random() self.log.debug("New auth key: %s" % self.settings["auth_key"]) self.saveSettings() if not self.settings.get("wrapper_key"): # To auth websocket permissions self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) self.saveSettings() self.websockets = [] # Active site websocket connections def __str__(self): return "Site %s" % self.address_short def __repr__(self): return "<%s>" % self.__str__() # Load site settings from data/sites.json def loadSettings(self): sites_settings = json.load(open("%s/sites.json" % config.data_dir)) if self.address in sites_settings: self.settings = sites_settings[self.address] else: self.settings = {"own": False, "serving": True, "permissions": [], "added": int(time.time())} # Default # Add admin permissions to homepage if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]: self.settings["permissions"].append("ADMIN") return # Save site settings to data/sites.json def saveSettings(self): sites_settings = json.load(open("%s/sites.json" % config.data_dir)) sites_settings[self.address] = self.settings helper.atomicWrite("%s/sites.json" % config.data_dir, json.dumps(sites_settings, indent=2, sort_keys=True)) # Max site size in MB def getSizeLimit(self): return self.settings.get("size_limit", config.size_limit) # Next size limit based on current size def getNextSizeLimit(self): size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000] size = self.settings.get("size", 0) for size_limit in size_limits: if size * 1.2 < size_limit * 1024 * 1024: return size_limit return 999999 # Download all file from content.json def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): s = time.time() if config.verbose: self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications") return False # Could not download content.json if config.verbose: self.log.debug("Got %s" % inner_path) changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False) if peer: # Update last received update from peer to prevent re-sending the same update to it peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"] # Start download files file_threads = [] if download_files: for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Try to diff first diff_success = False diff_actions = diffs.get(file_relative_path) if diff_actions and self.bad_files.get(file_inner_path): try: new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions) new_file.seek(0) diff_success = self.content_manager.verifyFile(file_inner_path, new_file) if diff_success: self.log.debug("Patched successfully: %s" % file_inner_path) new_file.seek(0) self.storage.write(file_inner_path, new_file) self.onFileDone(file_inner_path) except Exception, err: self.log.debug("Failed to patch %s: %s" % (file_inner_path, err)) diff_success = False if not diff_success: # Start download and dont wait for finish, return the event res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Optionals files if inner_path == "content.json": gevent.spawn(self.updateHashfield) if self.settings.get("autodownloadoptional"): for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Start download and dont wait for finish, return the event res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Wait for includes download include_threads = [] for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys(): file_inner_path = content_inner_dir + file_relative_path include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) include_threads.append(include_thread) if config.verbose: self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads))) gevent.joinall(include_threads) if config.verbose: self.log.debug("%s: Includes download ended" % inner_path) if check_modifications: # Check if every file is up-to-date self.checkModifications(0) if config.verbose: self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) gevent.joinall(file_threads) if config.verbose: self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s)) if not self.worker_manager.tasks: self.onComplete() # No more task trigger site complete return True
class Site(object): def __init__(self, address, allow_create=True): self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address self.address_short = "%s..%s" % (self.address[:6], self.address[-4:] ) # Short address for logging self.log = logging.getLogger("Site:%s" % self.address_short) self.addEventListeners() self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) self.time_announce = 0 # Last announce time to tracker self.last_tracker_id = random.randint(0, 10) # Last announced tracker id self.worker_manager = WorkerManager( self) # Handle site download from other peers self.bad_files = { } # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time self.notifications = [ ] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser self.storage = SiteStorage( self, allow_create=allow_create) # Save and load site files self.loadSettings() # Load settings from sites.json self.content_manager = ContentManager(self) # Load contents self.connection_server = None if "main" in sys.modules and "file_server" in dir( sys.modules["main"] ): # Use global file server by default if possible self.connection_server = sys.modules["main"].file_server else: self.connection_server = None if not self.settings.get( "auth_key" ): # To auth user in site (Obsolete, will be removed) self.settings["auth_key"] = CryptHash.random() self.log.debug("New auth key: %s" % self.settings["auth_key"]) self.saveSettings() if not self.settings.get( "wrapper_key"): # To auth websocket permissions self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) self.saveSettings() self.websockets = [] # Active site websocket connections def __str__(self): return "Site %s" % self.address_short def __repr__(self): return "<%s>" % self.__str__() # Load site settings from data/sites.json def loadSettings(self): sites_settings = json.load(open("%s/sites.json" % config.data_dir)) if self.address in sites_settings: self.settings = sites_settings[self.address] else: self.settings = { "own": False, "serving": True, "permissions": [], "added": int(time.time()) } # Default # Add admin permissions to homepage if self.address == config.homepage and "ADMIN" not in self.settings[ "permissions"]: self.settings["permissions"].append("ADMIN") return # Save site settings to data/sites.json def saveSettings(self): s = time.time() sites_settings = json.load(open("%s/sites.json" % config.data_dir)) sites_settings[self.address] = self.settings helper.atomicWrite( "%s/sites.json" % config.data_dir, json.dumps(sites_settings, indent=2, sort_keys=True)) self.log.debug("Saved settings in %.2fs" % (time.time() - s)) # Max site size in MB def getSizeLimit(self): return self.settings.get("size_limit", config.size_limit) # Next size limit based on current size def getNextSizeLimit(self): size_limits = [ 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000 ] size = self.settings.get("size", 0) for size_limit in size_limits: if size * 1.2 < size_limit * 1024 * 1024: return size_limit return 999999 # Download all file from content.json def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): s = time.time() if config.verbose: self.log.debug("Downloading %s..." % inner_path) found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later self.onFileDone.once( lambda file_name: self.checkModifications(0), "check_modifications") return False # Could not download content.json if config.verbose: self.log.debug("Got %s" % inner_path) changed, deleted = self.content_manager.loadContent( inner_path, load_includes=False) if peer: # Update last received update from peer to prevent re-sending the same update to it peer.last_content_json_update = self.content_manager.contents[ inner_path]["modified"] # Start download files file_threads = [] if download_files: for file_relative_path in self.content_manager.contents[ inner_path].get("files", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Try to diff first diff_success = False diff_actions = diffs.get(file_relative_path) if diff_actions and self.bad_files.get(file_inner_path): try: new_file = Diff.patch( self.storage.open(file_inner_path, "rb"), diff_actions) new_file.seek(0) diff_success = self.content_manager.verifyFile( file_inner_path, new_file) if diff_success: self.log.debug("Patched successfully: %s" % file_inner_path) new_file.seek(0) self.storage.write(file_inner_path, new_file) self.onFileDone(file_inner_path) except Exception, err: self.log.debug("Failed to patch %s: %s" % (file_inner_path, err)) diff_success = False if not diff_success: # Start download and dont wait for finish, return the event res = self.needFile( file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Optionals files if inner_path == "content.json": gevent.spawn(self.updateHashfield) if self.settings.get("autodownloadoptional"): for file_relative_path in self.content_manager.contents[ inner_path].get("files_optional", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Start download and dont wait for finish, return the event res = self.needFile( file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Wait for includes download include_threads = [] for file_relative_path in self.content_manager.contents[ inner_path].get("includes", {}).keys(): file_inner_path = content_inner_dir + file_relative_path include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) include_threads.append(include_thread) if config.verbose: self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads))) gevent.joinall(include_threads) if config.verbose: self.log.debug("%s: Includes download ended" % inner_path) if check_modifications: # Check if every file is up-to-date self.checkModifications(0) if config.verbose: self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) gevent.joinall(file_threads) if config.verbose: self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s)) if not self.worker_manager.tasks: self.onComplete() # No more task trigger site complete return True
class Site(object): def __init__(self, address, allow_create=True, settings=None): self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address self.address_hash = hashlib.sha256(self.address).digest() self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging self.log = logging.getLogger("Site:%s" % self.address_short) self.addEventListeners() self.content = None # Load content.json self.peers = {} # Key: ip:port, Value: Peer.Peer self.peers_recent = collections.deque(maxlen=100) self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself) self.worker_manager = WorkerManager(self) # Handle site download from other peers self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept) self.content_updated = None # Content.js update time self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout] self.page_requested = False # Page viewed in browser self.websockets = [] # Active site websocket connections self.connection_server = None self.loadSettings(settings) # Load settings from sites.json self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files self.content_manager = ContentManager(self) self.content_manager.loadContents() # Load content.json files if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible self.connection_server = sys.modules["main"].file_server else: self.log.debug("Creating connection server") # remove self.connection_server = FileServer() self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed) self.settings["auth_key"] = CryptHash.random() self.log.debug("New auth key: %s" % self.settings["auth_key"]) if not self.settings.get("wrapper_key"): # To auth websocket permissions self.settings["wrapper_key"] = CryptHash.random() self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"]) if not self.settings.get("ajax_key"): # To auth websocket permissions self.settings["ajax_key"] = CryptHash.random() self.log.debug("New ajax key: %s" % self.settings["ajax_key"]) def __str__(self): return "Site %s" % self.address_short def __repr__(self): return "<%s>" % self.__str__() # Load site settings from data/sites.json def loadSettings(self, settings=None): if not settings: settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address) if settings: self.settings = settings if "cache" not in settings: settings["cache"] = {} if "size_files_optional" not in settings: settings["size_optional"] = 0 if "optional_downloaded" not in settings: settings["optional_downloaded"] = 0 if "downloaded" not in settings: settings["downloaded"] = settings.get("added") self.bad_files = settings["cache"].get("bad_files", {}) settings["cache"]["bad_files"] = {} # Give it minimum 10 tries after restart for inner_path in self.bad_files: self.bad_files[inner_path] = min(self.bad_files[inner_path], 20) else: self.settings = { "own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0, "added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0 } # Default if config.download_optional == "auto": self.settings["autodownloadoptional"] = True # Add admin permissions to homepage if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]: self.settings["permissions"].append("ADMIN") return # Save site settings to data/sites.json def saveSettings(self): if not SiteManager.site_manager.sites: SiteManager.site_manager.sites = {} if not SiteManager.site_manager.sites.get(self.address): SiteManager.site_manager.sites[self.address] = self SiteManager.site_manager.load(False) SiteManager.site_manager.save() def getSettingsCache(self): back = {} back["bad_files"] = self.bad_files back["hashfield"] = self.content_manager.hashfield.tostring().encode("base64") return back # Max site size in MB def getSizeLimit(self): return self.settings.get("size_limit", int(config.size_limit)) # Next size limit based on current size def getNextSizeLimit(self): size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000] size = self.settings.get("size", 0) for size_limit in size_limits: if size * 1.2 < size_limit * 1024 * 1024: return size_limit return 999999 # Download all file from content.json def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}): s = time.time() if config.verbose: self.log.debug("Downloading %s..." % inner_path) if not inner_path.endswith("content.json"): return False found = self.needFile(inner_path, update=self.bad_files.get(inner_path)) content_inner_dir = helper.getDirname(inner_path) if not found: self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications)) if check_modifications: # Download failed, but check modifications if its succed later self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications") return False # Could not download content.json if config.verbose: self.log.debug("Got %s" % inner_path) changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False) if inner_path == "content.json": self.saveSettings() if peer: # Update last received update from peer to prevent re-sending the same update to it peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"] # Start download files file_threads = [] if download_files: for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys(): file_inner_path = content_inner_dir + file_relative_path # Try to diff first diff_success = False diff_actions = diffs.get(file_relative_path) if diff_actions and self.bad_files.get(file_inner_path): try: s = time.time() new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions) new_file.seek(0) time_diff = time.time() - s s = time.time() diff_success = self.content_manager.verifyFile(file_inner_path, new_file) time_verify = time.time() - s if diff_success: s = time.time() new_file.seek(0) self.storage.write(file_inner_path, new_file) time_write = time.time() - s s = time.time() self.onFileDone(file_inner_path) time_on_done = time.time() - s self.log.debug( "Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" % (file_inner_path, time_diff, time_verify, time_write, time_on_done) ) except Exception, err: self.log.debug("Failed to patch %s: %s" % (file_inner_path, err)) diff_success = False if not diff_success: # Start download and dont wait for finish, return the event res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Optionals files if inner_path == "content.json": gevent.spawn(self.updateHashfield) for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys(): file_inner_path = content_inner_dir + file_relative_path if file_inner_path not in changed and not self.bad_files.get(file_inner_path): continue if not self.isDownloadable(file_inner_path): continue # Start download and dont wait for finish, return the event res = self.pooledNeedFile( file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer ) if res is not True and res is not False: # Need downloading and file is allowed file_threads.append(res) # Append evt # Wait for includes download include_threads = [] for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys(): file_inner_path = content_inner_dir + file_relative_path include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer) include_threads.append(include_thread) if config.verbose: self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads))) gevent.joinall(include_threads) if config.verbose: self.log.debug("%s: Includes download ended" % inner_path) if check_modifications: # Check if every file is up-to-date self.checkModifications(0) if config.verbose: self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed))) gevent.joinall(file_threads) if config.verbose: self.log.debug("%s: DownloadContent ended in %.3fs" % (inner_path, time.time() - s)) if not self.worker_manager.tasks: self.onComplete() # No more task trigger site complete return True