def load(self, cleanup=True):
self.log.debug("Loading sites...")
from Site import Site
if self.sites is None:
self.sites = {}
address_found = []
added = 0
# Load new adresses
for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
s = time.time()
self.sites[address] = Site(address, settings=settings)
self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
added += 1
address_found.append(address)
# Remove deleted adresses
if cleanup:
for address in self.sites.keys():
if address not in address_found:
del(self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
for row in ContentDb.getContentDb().execute("SELECT * FROM site"):
if row["address"] not in self.sites:
self.log.info("Deleting orphan site from content.db: %s" % row["address"])
ContentDb.getContentDb().deleteSite(row["address"])
if added:
self.log.debug("SiteManager added %s sites" % added)
def load(self):
self.log.debug("Loading sites...")
from Site import Site
if not self.sites:
self.sites = {}
address_found = []
added = 0
# Load new adresses
for address, settings in json.load(
open("%s/sites.json" % config.data_dir)).iteritems():
if address not in self.sites and os.path.isfile(
"%s/%s/content.json" % (config.data_dir, address)):
s = time.time()
self.sites[address] = Site(address, settings=settings)
self.log.debug("Loaded site %s in %.3fs" %
(address, time.time() - s))
added += 1
address_found.append(address)
# Remove deleted adresses
for address in self.sites.keys():
if address not in address_found:
del (self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
for row in ContentDb.getContentDb().execute("SELECT * FROM site"):
if row["address"] not in self.sites:
self.log.info("Deleting orphan site from content.db: %s" %
row["address"])
ContentDb.getContentDb().deleteSite(row["address"])
if added:
self.log.debug("SiteManager added %s sites" % added)
class SiteManager(object):
def __init__(self):
self.log = logging.getLogger("SiteManager")
self.log.debug("SiteManager created.")
self.sites = None
self.loaded = False
gevent.spawn(self.saveTimer)
atexit.register(self.save)
# Load all sites from data/sites.json
def load(self, cleanup=True):
self.log.debug("Loading sites...")
self.loaded = False
from Site import Site
if self.sites is None:
self.sites = {}
address_found = []
added = 0
# Load new adresses
for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
s = time.time()
try:
site = Site(address, settings=settings)
site.content_manager.contents.get("content.json")
except Exception, err:
self.log.debug("Error loading site %s: %s" % (address, err))
continue
self.sites[address] = site
self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
added += 1
address_found.append(address)
# Remove deleted adresses
if cleanup:
for address in self.sites.keys():
if address not in address_found:
del(self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
content_db = ContentDb.getContentDb()
for row in content_db.execute("SELECT * FROM site"):
address = row["address"]
if address not in self.sites:
self.log.info("Deleting orphan site from content.db: %s" % address)
content_db.execute("DELETE FROM site WHERE ?", {"address": address})
if address in content_db.site_ids:
del content_db.site_ids[address]
if address in content_db.sites:
del content_db.sites[address]
if added:
self.log.debug("SiteManager added %s sites" % added)
self.loaded = True
def load(self, cleanup=True):
self.log.debug("Loading sites...")
self.loaded = False
from Site import Site
if self.sites is None:
self.sites = {}
address_found = []
added = 0
# Load new adresses
for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
s = time.time()
self.sites[address] = Site(address, settings=settings)
self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
added += 1
address_found.append(address)
# Remove deleted adresses
if cleanup:
for address in self.sites.keys():
if address not in address_found:
del(self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
content_db = ContentDb.getContentDb()
for row in content_db.execute("SELECT * FROM site"):
address = row["address"]
if address not in self.sites:
self.log.info("Deleting orphan site from content.db: %s" % address)
content_db.execute("DELETE FROM site WHERE ?", {"address": address})
if address in content_db.site_ids:
del content_db.site_ids[address]
if address in content_db.sites:
del content_db.sites[address]
if added:
self.log.debug("SiteManager added %s sites" % added)
self.loaded = True
def load(self, cleanup=True, startup=False):
from Debug import Debug
self.log.info("Loading sites... (cleanup: %s, startup: %s)" %
(cleanup, startup))
self.loaded = False
from .Site import Site
address_found = []
added = 0
load_s = time.time()
# Load new adresses
try:
json_path = "%s/sites.json" % config.data_dir
data = json.load(open(json_path))
except Exception as err:
raise Exception("Unable to load %s: %s" % (json_path, err))
sites_need = []
for address, settings in data.items():
if address not in self.sites:
if os.path.isfile("%s/%s/content.json" %
(config.data_dir, address)):
# Root content.json exists, try load site
s = time.time()
try:
site = Site(address, settings=settings)
site.content_manager.contents.get("content.json")
except Exception as err:
self.log.debug("Error loading site %s: %s" %
(address, err))
continue
self.sites[address] = site
self.log.debug("Loaded site %s in %.3fs" %
(address, time.time() - s))
added += 1
elif startup:
# No site directory, start download
self.log.debug("Found new site in sites.json: %s" %
address)
sites_need.append([address, settings])
added += 1
address_found.append(address)
# Remove deleted adresses
if cleanup:
for address in list(self.sites.keys()):
if address not in address_found:
del (self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
content_db = ContentDb.getContentDb()
for row in content_db.execute("SELECT * FROM site").fetchall():
address = row["address"]
if address not in self.sites and address not in address_found:
self.log.info("Deleting orphan site from content.db: %s" %
address)
try:
content_db.execute("DELETE FROM site WHERE ?",
{"address": address})
except Exception as err:
self.log.error(
"Can't delete site %s from content_db: %s" %
(address, err))
if address in content_db.site_ids:
del content_db.site_ids[address]
if address in content_db.sites:
del content_db.sites[address]
self.loaded = True
for address, settings in sites_need:
gevent.spawn(self.need, address, settings=settings)
if added:
self.log.info("Added %s sites in %.3fs" %
(added, time.time() - load_s))
class SiteManager(object):
def __init__(self):
self.log = logging.getLogger("SiteManager")
self.log.debug("SiteManager created.")
self.sites = {}
self.sites_changed = int(time.time())
self.loaded = False
gevent.spawn(self.saveTimer)
atexit.register(lambda: self.save(recalculate_size=True))
# Load all sites from data/sites.json
def load(self, cleanup=True, startup=False):
self.log.debug("Loading sites...")
self.loaded = False
from Site import Site
address_found = []
added = 0
# Load new adresses
for address, settings in json.load(
open("%s/sites.json" % config.data_dir)).iteritems():
if address not in self.sites:
if os.path.isfile("%s/%s/content.json" %
(config.data_dir, address)):
# Root content.json exists, try load site
s = time.time()
try:
site = Site(address, settings=settings)
site.content_manager.contents.get("content.json")
except Exception, err:
self.log.debug("Error loading site %s: %s" %
(address, err))
continue
self.sites[address] = site
self.log.debug("Loaded site %s in %.3fs" %
(address, time.time() - s))
added += 1
elif startup and (settings.get("peers", 0) > 0
or settings.get("own")):
# No site directory, start download
self.log.debug("Found new site in sites.json: %s" %
address)
gevent.spawn(self.need, address, settings=settings)
added += 1
address_found.append(address)
# Remove deleted adresses
if cleanup:
for address in self.sites.keys():
if address not in address_found:
del (self.sites[address])
self.log.debug("Removed site: %s" % address)
# Remove orpan sites from contentdb
content_db = ContentDb.getContentDb()
for row in content_db.execute("SELECT * FROM site").fetchall():
address = row["address"]
if address not in self.sites:
self.log.info("Deleting orphan site from content.db: %s" %
address)
try:
content_db.execute("DELETE FROM site WHERE ?",
{"address": address})
except Exception as err:
self.log.error(
"Can't delete site %s from content_db: %s" %
(address, err))
if address in content_db.site_ids:
del content_db.site_ids[address]
if address in content_db.sites:
del content_db.sites[address]
if added:
self.log.debug("SiteManager added %s sites" % added)
self.loaded = True