class Site(object):
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.time_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0,
10) # Last announced tracker id
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {
} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(
self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(
sys.modules["main"]
): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
self.settings = {
"own": False,
"serving": True,
"permissions": [],
"added": int(time.time())
} # Default
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings[
"permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
s = time.time()
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite(
"%s/sites.json" % config.data_dir,
json.dumps(sites_settings, indent=2, sort_keys=True))
self.log.debug("Saved settings in %.2fs" % (time.time() - s))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self,
inner_path,
download_files=True,
peer=None,
check_modifications=False,
diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" %
(inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(
lambda file_name: self.checkModifications(0),
"check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(
inner_path, load_includes=False)
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[
inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
new_file = Diff.patch(
self.storage.open(file_inner_path, "rb"),
diff_actions)
new_file.seek(0)
diff_success = self.content_manager.verifyFile(
file_inner_path, new_file)
if diff_success:
self.log.debug("Patched successfully: %s" %
file_inner_path)
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
self.onFileDone(file_inner_path)
except Exception, err:
self.log.debug("Failed to patch %s: %s" %
(file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[
inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent,
file_inner_path,
download_files=download_files,
peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." %
(inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." %
(inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.2fs" %
(inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site(object):
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.time_announce = 0 # Last announce time to tracker
self.last_tracker_id = random.randint(0, 10) # Last announced tracker id
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
self.connection_server = None
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.connection_server = None
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
self.settings = {"own": False, "serving": True, "permissions": [], "added": int(time.time())} # Default
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("%s/sites.json" % config.data_dir))
sites_settings[self.address] = self.settings
helper.atomicWrite("%s/sites.json" % config.data_dir, json.dumps(sites_settings, indent=2, sort_keys=True))
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
new_file.seek(0)
diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
if diff_success:
self.log.debug("Patched successfully: %s" % file_inner_path)
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
self.onFileDone(file_inner_path)
except Exception, err:
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
if self.settings.get("autodownloadoptional"):
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.2fs" % (inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site(object):
def __init__(self, address, allow_create=True, settings=None):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_hash = hashlib.sha256(self.address).digest()
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.addEventListeners()
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peers_recent = collections.deque(maxlen=100)
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.websockets = [] # Active site websocket connections
self.connection_server = None
self.loadSettings(settings) # Load settings from sites.json
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.content_manager = ContentManager(self)
self.content_manager.loadContents() # Load content.json files
if "main" in sys.modules and "file_server" in dir(sys.modules["main"]): # Use global file server by default if possible
self.connection_server = sys.modules["main"].file_server
else:
self.log.debug("Creating connection server") # remove
self.connection_server = FileServer()
self.announcer = SiteAnnouncer(self) # Announce and get peer list from other nodes
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = CryptHash.random()
self.log.debug("New auth key: %s" % self.settings["auth_key"])
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = CryptHash.random()
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
if not self.settings.get("ajax_key"): # To auth websocket permissions
self.settings["ajax_key"] = CryptHash.random()
self.log.debug("New ajax key: %s" % self.settings["ajax_key"])
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self, settings=None):
if not settings:
settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
if settings:
self.settings = settings
if "cache" not in settings:
settings["cache"] = {}
if "size_files_optional" not in settings:
settings["size_optional"] = 0
if "optional_downloaded" not in settings:
settings["optional_downloaded"] = 0
if "downloaded" not in settings:
settings["downloaded"] = settings.get("added")
self.bad_files = settings["cache"].get("bad_files", {})
settings["cache"]["bad_files"] = {}
# Give it minimum 10 tries after restart
for inner_path in self.bad_files:
self.bad_files[inner_path] = min(self.bad_files[inner_path], 20)
else:
self.settings = {
"own": False, "serving": True, "permissions": [], "cache": {"bad_files": {}}, "size_files_optional": 0,
"added": int(time.time()), "downloaded": None, "optional_downloaded": 0, "size_optional": 0
} # Default
if config.download_optional == "auto":
self.settings["autodownloadoptional"] = True
# Add admin permissions to homepage
if self.address == config.homepage and "ADMIN" not in self.settings["permissions"]:
self.settings["permissions"].append("ADMIN")
return
# Save site settings to data/sites.json
def saveSettings(self):
if not SiteManager.site_manager.sites:
SiteManager.site_manager.sites = {}
if not SiteManager.site_manager.sites.get(self.address):
SiteManager.site_manager.sites[self.address] = self
SiteManager.site_manager.load(False)
SiteManager.site_manager.save()
def getSettingsCache(self):
back = {}
back["bad_files"] = self.bad_files
back["hashfield"] = self.content_manager.hashfield.tostring().encode("base64")
return back
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", int(config.size_limit))
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
def downloadContent(self, inner_path, download_files=True, peer=None, check_modifications=False, diffs={}):
s = time.time()
if config.verbose:
self.log.debug("Downloading %s..." % inner_path)
if not inner_path.endswith("content.json"):
return False
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = helper.getDirname(inner_path)
if not found:
self.log.debug("Download %s failed, check_modifications: %s" % (inner_path, check_modifications))
if check_modifications: # Download failed, but check modifications if its succed later
self.onFileDone.once(lambda file_name: self.checkModifications(0), "check_modifications")
return False # Could not download content.json
if config.verbose:
self.log.debug("Got %s" % inner_path)
changed, deleted = self.content_manager.loadContent(inner_path, load_includes=False)
if inner_path == "content.json":
self.saveSettings()
if peer: # Update last received update from peer to prevent re-sending the same update to it
peer.last_content_json_update = self.content_manager.contents[inner_path]["modified"]
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
# Try to diff first
diff_success = False
diff_actions = diffs.get(file_relative_path)
if diff_actions and self.bad_files.get(file_inner_path):
try:
s = time.time()
new_file = Diff.patch(self.storage.open(file_inner_path, "rb"), diff_actions)
new_file.seek(0)
time_diff = time.time() - s
s = time.time()
diff_success = self.content_manager.verifyFile(file_inner_path, new_file)
time_verify = time.time() - s
if diff_success:
s = time.time()
new_file.seek(0)
self.storage.write(file_inner_path, new_file)
time_write = time.time() - s
s = time.time()
self.onFileDone(file_inner_path)
time_on_done = time.time() - s
self.log.debug(
"Patched successfully: %s (diff: %.3fs, verify: %.3fs, write: %.3fs, on_done: %.3fs)" %
(file_inner_path, time_diff, time_verify, time_write, time_on_done)
)
except Exception, err:
self.log.debug("Failed to patch %s: %s" % (file_inner_path, err))
diff_success = False
if not diff_success:
# Start download and dont wait for finish, return the event
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Optionals files
if inner_path == "content.json":
gevent.spawn(self.updateHashfield)
for file_relative_path in self.content_manager.contents[inner_path].get("files_optional", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
if file_inner_path not in changed and not self.bad_files.get(file_inner_path):
continue
if not self.isDownloadable(file_inner_path):
continue
# Start download and dont wait for finish, return the event
res = self.pooledNeedFile(
file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer
)
if res is not True and res is not False: # Need downloading and file is allowed
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
if config.verbose:
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
if config.verbose:
self.log.debug("%s: Includes download ended" % inner_path)
if check_modifications: # Check if every file is up-to-date
self.checkModifications(0)
if config.verbose:
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
if config.verbose:
self.log.debug("%s: DownloadContent ended in %.3fs" % (inner_path, time.time() - s))
if not self.worker_manager.tasks:
self.onComplete() # No more task trigger site complete
return True
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "", address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(self) # Handle site download from other peers
self.bad_files = {} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get("auth_key"): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get("wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase + string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" % self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = { "own": False, "serving": True, "permissions": permissions } # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json", "w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [10,20,50,100,200,500,1000,2000,5000,10000,20000,50000,100000]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size*1.2 < size_limit*1024*1024:
return size_limit
return 999999
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path, update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path, load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
res = self.needFile(file_inner_path, blocking=False, update=self.bad_files.get(file_inner_path), peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir+file_relative_path
include_thread = gevent.spawn(self.downloadContent, file_inner_path, download_files=download_files, peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." % (inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." % (inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" % (inner_path, time.time()-s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files: return False
return [bad_file for bad_file, retry in self.bad_files.iteritems() if retry < 3]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid: return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update content.json from peers and download changed files
@util.Noparallel()
def update(self):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." % len(content_threads))
gevent.joinall(content_threads)
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = self.bad_files.get(changed_file, 0)+1
if not self.settings["own"]: self.storage.checkFiles(quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize() # Update site size
return changed
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
timeout = 5+int(self.storage.getSize(inner_path)/1024) # Timeout: 5sec + size in kb
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request("update", {
"site": self.address,
"inner_path": inner_path,
"body": self.storage.open(inner_path).read(),
"peer": (config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
self.log.info("[FAILED] %s: %s" % (peer.key, result))
class Site:
def __init__(self, address, allow_create=True):
self.address = re.sub("[^A-Za-z0-9]", "",
address) # Make sure its correct address
self.address_short = "%s..%s" % (self.address[:6], self.address[-4:]
) # Short address for logging
self.log = logging.getLogger("Site:%s" % self.address_short)
self.content = None # Load content.json
self.peers = {} # Key: ip:port, Value: Peer.Peer
self.peer_blacklist = SiteManager.peer_blacklist # Ignore this peers (eg. myself)
self.last_announce = 0 # Last announce time to tracker
self.worker_manager = WorkerManager(
self) # Handle site download from other peers
self.bad_files = {
} # SHA512 check failed files, need to redownload {"inner.content": 1} (key: file, value: failed accept)
self.content_updated = None # Content.js update time
self.notifications = [
] # Pending notifications displayed once on page load [error|ok|info, message, timeout]
self.page_requested = False # Page viewed in browser
self.storage = SiteStorage(
self, allow_create=allow_create) # Save and load site files
self.loadSettings() # Load settings from sites.json
self.content_manager = ContentManager(self) # Load contents
if not self.settings.get(
"auth_key"
): # To auth user in site (Obsolete, will be removed)
self.settings["auth_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(24))
self.log.debug("New auth key: %s" % self.settings["auth_key"])
self.saveSettings()
if not self.settings.get(
"wrapper_key"): # To auth websocket permissions
self.settings["wrapper_key"] = ''.join(
random.choice(string.ascii_uppercase + string.ascii_lowercase +
string.digits) for _ in range(12))
self.log.debug("New wrapper key: %s" %
self.settings["wrapper_key"])
self.saveSettings()
self.websockets = [] # Active site websocket connections
# Add event listeners
self.addEventListeners()
def __str__(self):
return "Site %s" % self.address_short
def __repr__(self):
return "<%s>" % self.__str__()
# Load site settings from data/sites.json
def loadSettings(self):
sites_settings = json.load(open("data/sites.json"))
if self.address in sites_settings:
self.settings = sites_settings[self.address]
else:
if self.address == config.homepage: # Add admin permissions to homepage
permissions = ["ADMIN"]
else:
permissions = []
self.settings = {
"own": False,
"serving": True,
"permissions": permissions
} # Default
return
# Save site settings to data/sites.json
def saveSettings(self):
sites_settings = json.load(open("data/sites.json"))
sites_settings[self.address] = self.settings
open("data/sites.json",
"w").write(json.dumps(sites_settings, indent=2, sort_keys=True))
return
# Max site size in MB
def getSizeLimit(self):
return self.settings.get("size_limit", config.size_limit)
# Next size limit based on current size
def getNextSizeLimit(self):
size_limits = [
10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000,
100000
]
size = self.settings.get("size", 0)
for size_limit in size_limits:
if size * 1.2 < size_limit * 1024 * 1024:
return size_limit
return 999999
# Download all file from content.json
@util.Noparallel(blocking=True)
def downloadContent(self, inner_path, download_files=True, peer=None):
s = time.time()
self.log.debug("Downloading %s..." % inner_path)
found = self.needFile(inner_path,
update=self.bad_files.get(inner_path))
content_inner_dir = self.content_manager.toDir(inner_path)
if not found: return False # Could not download content.json
self.log.debug("Got %s" % inner_path)
changed = self.content_manager.loadContent(inner_path,
load_includes=False)
# Start download files
file_threads = []
if download_files:
for file_relative_path in self.content_manager.contents[
inner_path].get("files", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
res = self.needFile(
file_inner_path,
blocking=False,
update=self.bad_files.get(file_inner_path),
peer=peer) # No waiting for finish, return the event
if res != True: # Need downloading
file_threads.append(res) # Append evt
# Wait for includes download
include_threads = []
for file_relative_path in self.content_manager.contents[
inner_path].get("includes", {}).keys():
file_inner_path = content_inner_dir + file_relative_path
include_thread = gevent.spawn(self.downloadContent,
file_inner_path,
download_files=download_files,
peer=peer)
include_threads.append(include_thread)
self.log.debug("%s: Downloading %s includes..." %
(inner_path, len(include_threads)))
gevent.joinall(include_threads)
self.log.debug("%s: Includes downloaded" % inner_path)
self.log.debug("%s: Downloading %s files, changed: %s..." %
(inner_path, len(file_threads), len(changed)))
gevent.joinall(file_threads)
self.log.debug("%s: All file downloaded in %.2fs" %
(inner_path, time.time() - s))
return True
# Return bad files with less than 3 retry
def getReachableBadFiles(self):
if not self.bad_files: return False
return [
bad_file for bad_file, retry in self.bad_files.iteritems()
if retry < 3
]
# Retry download bad files
def retryBadFiles(self):
for bad_file in self.bad_files.keys():
self.needFile(bad_file, update=True, blocking=False)
# Download all files of the site
@util.Noparallel(blocking=False)
def download(self, check_size=False):
self.log.debug("Start downloading...%s" % self.bad_files)
self.announce()
if check_size: # Check the size first
valid = downloadContent(download_files=False)
if not valid:
return False # Cant download content.jsons or size is not fits
found = self.downloadContent("content.json")
return found
# Update content.json from peers and download changed files
@util.Noparallel()
def update(self):
self.content_manager.loadContent("content.json") # Reload content.json
self.content_updated = None
# Download all content.json again
content_threads = []
for inner_path in self.content_manager.contents.keys():
content_threads.append(
self.needFile(inner_path, update=True, blocking=False))
self.log.debug("Waiting %s content.json to finish..." %
len(content_threads))
gevent.joinall(content_threads)
changed = self.content_manager.loadContent("content.json")
if changed:
for changed_file in changed:
self.bad_files[changed_file] = self.bad_files.get(
changed_file, 0) + 1
if not self.settings["own"]:
self.storage.checkFiles(
quick_check=True) # Quick check files based on file size
if self.bad_files:
self.download()
self.settings["size"] = self.content_manager.getTotalSize(
) # Update site size
return changed
# Publish worker
def publisher(self, inner_path, peers, published, limit, event_done=None):
timeout = 5 + int(self.storage.getSize(inner_path) /
1024) # Timeout: 5sec + size in kb
while 1:
if not peers or len(published) >= limit:
if event_done: event_done.set(True)
break # All peers done, or published engouht
peer = peers.pop(0)
result = {"exception": "Timeout"}
for retry in range(2):
try:
with gevent.Timeout(timeout, False):
result = peer.request(
"update", {
"site":
self.address,
"inner_path":
inner_path,
"body":
self.storage.open(inner_path).read(),
"peer":
(config.ip_external, config.fileserver_port)
})
if result: break
except Exception, err:
result = {"exception": Debug.formatException(err)}
if result and "ok" in result:
published.append(peer)
self.log.info("[OK] %s: %s" % (peer.key, result["ok"]))
else:
self.log.info("[FAILED] %s: %s" % (peer.key, result))
