Esempio n. 1
0
    def load(self, cleanup=True):
        self.log.debug("Loading sites...")
        from Site import Site
        if self.sites is None:
            self.sites = {}
        address_found = []
        added = 0
        # Load new adresses
        for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
            if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
                s = time.time()
                self.sites[address] = Site(address, settings=settings)
                self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
                added += 1
            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in self.sites.keys():
                if address not in address_found:
                    del(self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            for row in ContentDb.getContentDb().execute("SELECT * FROM site"):
                if row["address"] not in self.sites:
                    self.log.info("Deleting orphan site from content.db: %s" % row["address"])
                    ContentDb.getContentDb().deleteSite(row["address"])

        if added:
            self.log.debug("SiteManager added %s sites" % added)
Esempio n. 2
0
    def load(self):
        self.log.debug("Loading sites...")
        from Site import Site
        if not self.sites:
            self.sites = {}
        address_found = []
        added = 0
        # Load new adresses
        for address, settings in json.load(
                open("%s/sites.json" % config.data_dir)).iteritems():
            if address not in self.sites and os.path.isfile(
                    "%s/%s/content.json" % (config.data_dir, address)):
                s = time.time()
                self.sites[address] = Site(address, settings=settings)
                self.log.debug("Loaded site %s in %.3fs" %
                               (address, time.time() - s))
                added += 1
            address_found.append(address)

        # Remove deleted adresses
        for address in self.sites.keys():
            if address not in address_found:
                del (self.sites[address])
                self.log.debug("Removed site: %s" % address)

        # Remove orpan sites from contentdb
        for row in ContentDb.getContentDb().execute("SELECT * FROM site"):
            if row["address"] not in self.sites:
                self.log.info("Deleting orphan site from content.db: %s" %
                              row["address"])
                ContentDb.getContentDb().deleteSite(row["address"])

        if added:
            self.log.debug("SiteManager added %s sites" % added)
Esempio n. 3
0
class SiteManager(object):
    def __init__(self):
        self.log = logging.getLogger("SiteManager")
        self.log.debug("SiteManager created.")
        self.sites = None
        self.loaded = False
        gevent.spawn(self.saveTimer)
        atexit.register(self.save)

    # Load all sites from data/sites.json
    def load(self, cleanup=True):
        self.log.debug("Loading sites...")
        self.loaded = False
        from Site import Site
        if self.sites is None:
            self.sites = {}
        address_found = []
        added = 0
        # Load new adresses
        for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
            if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
                s = time.time()
                try:
                    site = Site(address, settings=settings)
                    site.content_manager.contents.get("content.json")
                except Exception, err:
                    self.log.debug("Error loading site %s: %s" % (address, err))
                    continue
                self.sites[address] = site
                self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
                added += 1
            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in self.sites.keys():
                if address not in address_found:
                    del(self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            content_db = ContentDb.getContentDb()
            for row in content_db.execute("SELECT * FROM site"):
                address = row["address"]
                if address not in self.sites:
                    self.log.info("Deleting orphan site from content.db: %s" % address)
                    content_db.execute("DELETE FROM site WHERE ?", {"address": address})
                    if address in content_db.site_ids:
                        del content_db.site_ids[address]
                    if address in content_db.sites:
                        del content_db.sites[address]

        if added:
            self.log.debug("SiteManager added %s sites" % added)
        self.loaded = True
Esempio n. 4
0
    def load(self, cleanup=True):
        self.log.debug("Loading sites...")
        self.loaded = False
        from Site import Site
        if self.sites is None:
            self.sites = {}
        address_found = []
        added = 0
        # Load new adresses
        for address, settings in json.load(open("%s/sites.json" % config.data_dir)).iteritems():
            if address not in self.sites and os.path.isfile("%s/%s/content.json" % (config.data_dir, address)):
                s = time.time()
                self.sites[address] = Site(address, settings=settings)
                self.log.debug("Loaded site %s in %.3fs" % (address, time.time() - s))
                added += 1
            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in self.sites.keys():
                if address not in address_found:
                    del(self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            content_db = ContentDb.getContentDb()
            for row in content_db.execute("SELECT * FROM site"):
                address = row["address"]
                if address not in self.sites:
                    self.log.info("Deleting orphan site from content.db: %s" % address)
                    content_db.execute("DELETE FROM site WHERE ?", {"address": address})
                    if address in content_db.site_ids:
                        del content_db.site_ids[address]
                    if address in content_db.sites:
                        del content_db.sites[address]

        if added:
            self.log.debug("SiteManager added %s sites" % added)
        self.loaded = True
Esempio n. 5
0
    def load(self, cleanup=True, startup=False):
        from Debug import Debug
        self.log.info("Loading sites... (cleanup: %s, startup: %s)" %
                      (cleanup, startup))
        self.loaded = False
        from .Site import Site
        address_found = []
        added = 0
        load_s = time.time()
        # Load new adresses
        try:
            json_path = "%s/sites.json" % config.data_dir
            data = json.load(open(json_path))
        except Exception as err:
            raise Exception("Unable to load %s: %s" % (json_path, err))

        sites_need = []

        for address, settings in data.items():
            if address not in self.sites:
                if os.path.isfile("%s/%s/content.json" %
                                  (config.data_dir, address)):
                    # Root content.json exists, try load site
                    s = time.time()
                    try:
                        site = Site(address, settings=settings)
                        site.content_manager.contents.get("content.json")
                    except Exception as err:
                        self.log.debug("Error loading site %s: %s" %
                                       (address, err))
                        continue
                    self.sites[address] = site
                    self.log.debug("Loaded site %s in %.3fs" %
                                   (address, time.time() - s))
                    added += 1
                elif startup:
                    # No site directory, start download
                    self.log.debug("Found new site in sites.json: %s" %
                                   address)
                    sites_need.append([address, settings])
                    added += 1

            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in list(self.sites.keys()):
                if address not in address_found:
                    del (self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            content_db = ContentDb.getContentDb()
            for row in content_db.execute("SELECT * FROM site").fetchall():
                address = row["address"]
                if address not in self.sites and address not in address_found:
                    self.log.info("Deleting orphan site from content.db: %s" %
                                  address)

                    try:
                        content_db.execute("DELETE FROM site WHERE ?",
                                           {"address": address})
                    except Exception as err:
                        self.log.error(
                            "Can't delete site %s from content_db: %s" %
                            (address, err))

                    if address in content_db.site_ids:
                        del content_db.site_ids[address]
                    if address in content_db.sites:
                        del content_db.sites[address]

        self.loaded = True
        for address, settings in sites_need:
            gevent.spawn(self.need, address, settings=settings)
        if added:
            self.log.info("Added %s sites in %.3fs" %
                          (added, time.time() - load_s))
Esempio n. 6
0
class SiteManager(object):
    def __init__(self):
        self.log = logging.getLogger("SiteManager")
        self.log.debug("SiteManager created.")
        self.sites = {}
        self.sites_changed = int(time.time())
        self.loaded = False
        gevent.spawn(self.saveTimer)
        atexit.register(lambda: self.save(recalculate_size=True))

    # Load all sites from data/sites.json
    def load(self, cleanup=True, startup=False):
        self.log.debug("Loading sites...")
        self.loaded = False
        from Site import Site
        address_found = []
        added = 0
        # Load new adresses
        for address, settings in json.load(
                open("%s/sites.json" % config.data_dir)).iteritems():
            if address not in self.sites:
                if os.path.isfile("%s/%s/content.json" %
                                  (config.data_dir, address)):
                    # Root content.json exists, try load site
                    s = time.time()
                    try:
                        site = Site(address, settings=settings)
                        site.content_manager.contents.get("content.json")
                    except Exception, err:
                        self.log.debug("Error loading site %s: %s" %
                                       (address, err))
                        continue
                    self.sites[address] = site
                    self.log.debug("Loaded site %s in %.3fs" %
                                   (address, time.time() - s))
                    added += 1
                elif startup and (settings.get("peers", 0) > 0
                                  or settings.get("own")):
                    # No site directory, start download
                    self.log.debug("Found new site in sites.json: %s" %
                                   address)
                    gevent.spawn(self.need, address, settings=settings)
                    added += 1

            address_found.append(address)

        # Remove deleted adresses
        if cleanup:
            for address in self.sites.keys():
                if address not in address_found:
                    del (self.sites[address])
                    self.log.debug("Removed site: %s" % address)

            # Remove orpan sites from contentdb
            content_db = ContentDb.getContentDb()
            for row in content_db.execute("SELECT * FROM site").fetchall():
                address = row["address"]
                if address not in self.sites:
                    self.log.info("Deleting orphan site from content.db: %s" %
                                  address)

                    try:
                        content_db.execute("DELETE FROM site WHERE ?",
                                           {"address": address})
                    except Exception as err:
                        self.log.error(
                            "Can't delete site %s from content_db: %s" %
                            (address, err))

                    if address in content_db.site_ids:
                        del content_db.site_ids[address]
                    if address in content_db.sites:
                        del content_db.sites[address]

        if added:
            self.log.debug("SiteManager added %s sites" % added)
        self.loaded = True