Esempio n. 1
0
    def siteDownload(self, address):
        from Site import Site
        from Site import SiteManager
        SiteManager.site_manager.load()

        logging.info("Opening a simple connection server")
        global file_server
        from Connection import ConnectionServer
        file_server = ConnectionServer("127.0.0.1", 1234)

        site = Site(address)

        on_completed = gevent.event.AsyncResult()

        def onComplete(evt):
            evt.set(True)

        site.onComplete.once(lambda: onComplete(on_completed))
        print "Announcing..."
        site.announce()

        s = time.time()
        print "Downloading..."
        site.downloadContent("content.json", check_modifications=True)

        print on_completed.get()
        print "Downloaded in %.3fs" % (time.time()-s)
Esempio n. 2
0
    def sitePublish(self,
                    address,
                    peer_ip=None,
                    peer_port=15441,
                    inner_path="content.json",
                    diffs={}):
        global file_server
        from Site import Site
        from File import FileServer  # We need fileserver to handle incoming file requests
        from Peer import Peer

        logging.info("Loading site...")
        site = Site(address, allow_create=False)
        site.settings[
            "serving"] = True  # Serving the site even if its disabled

        logging.info("Creating FileServer....")
        file_server = FileServer()
        site.connection_server = file_server
        file_server_thread = gevent.spawn(
            file_server.start,
            check_sites=False)  # Dont check every site integrity
        time.sleep(0.001)

        if not file_server_thread.ready():
            # Started fileserver
            file_server.openport()
            if peer_ip:  # Announce ip specificed
                site.addPeer(peer_ip, peer_port)
            else:  # Just ask the tracker
                logging.info("Gathering peers from tracker")
                site.announce()  # Gather peers
            published = site.publish(5, inner_path,
                                     diffs=diffs)  # Push to peers
            if published > 0:
                time.sleep(3)
                logging.info("Serving files (max 60s)...")
                gevent.joinall([file_server_thread], timeout=60)
                logging.info("Done.")
            else:
                logging.info(
                    "No peers found, sitePublish command only works if you already have visitors serving your site"
                )
        else:
            # Already running, notify local client on new content
            logging.info("Sending siteReload")
            my_peer = Peer("127.0.0.1", config.fileserver_port)
            logging.info(
                my_peer.request("siteReload", {
                    "site": site.address,
                    "inner_path": inner_path
                }))
            logging.info("Sending sitePublish")
            logging.info(
                my_peer.request("sitePublish", {
                    "site": site.address,
                    "inner_path": inner_path,
                    "diffs": diffs
                }))
            logging.info("Done.")
Esempio n. 3
0
    def siteAnnounce(self, address):
        from Site.Site import Site
        logging.info("Announcing site %s to tracker..." % address)
        site = Site(address)

        s = time.time()
        site.announce()
        print "Response time: %.3fs" % (time.time() - s)
        print site.peers
Esempio n. 4
0
    def siteAnnounce(self, address):
        from Site.Site import Site
        logging.info("Announcing site %s to tracker..." % address)
        site = Site(address)

        s = time.time()
        site.announce()
        print "Response time: %.3fs" % (time.time() - s)
        print site.peers
Esempio n. 5
0
def site(request):
    threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)]
    # Reset ratelimit
    RateLimit.queue_db = {}
    RateLimit.called_db = {}

    site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")

    # Always use original data
    assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in site.storage.getPath("")  # Make sure we dont delete everything
    shutil.rmtree(site.storage.getPath(""), True)
    shutil.copytree(site.storage.getPath("") + "-original", site.storage.getPath(""))

    # Add to site manager
    SiteManager.site_manager.get("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
    site.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site.storage.deleteFiles()
        site.content_manager.contents.db.deleteSite(site)
        del SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"]
        site.content_manager.contents.db.close()
        SiteManager.site_manager.sites.clear()
        db_path = "%s/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]
        gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before])
    request.addfinalizer(cleanup)

    site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")  # Create new Site object to load content.json files
    if not SiteManager.site_manager.sites:
        SiteManager.site_manager.sites = {}
    SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] = site
    return site
Esempio n. 6
0
def site(request):
    threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)]
    # Reset ratelimit
    RateLimit.queue_db = {}
    RateLimit.called_db = {}

    site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")

    # Always use original data
    assert "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT" in site.storage.getPath("")  # Make sure we dont delete everything
    shutil.rmtree(site.storage.getPath(""), True)
    shutil.copytree(site.storage.getPath("") + "-original", site.storage.getPath(""))

    # Add to site manager
    SiteManager.site_manager.get("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
    site.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site.storage.deleteFiles()
        site.content_manager.contents.db.deleteSite(site)
        del SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"]
        site.content_manager.contents.db.close()
        SiteManager.site_manager.sites.clear()
        db_path = "%s/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]
        gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before])
    request.addfinalizer(cleanup)

    site = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")  # Create new Site object to load content.json files
    if not SiteManager.site_manager.sites:
        SiteManager.site_manager.sites = {}
    SiteManager.site_manager.sites["1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT"] = site
    return site
Esempio n. 7
0
    def siteAnnounce(self, address):
        from Site.Site import Site
        from Site import SiteManager
        SiteManager.site_manager.load()

        logging.info("Opening a simple connection server")
        global file_server
        from File import FileServer
        file_server = FileServer("127.0.0.1", 1234)

        logging.info("Announcing site %s to tracker..." % address)
        site = Site(address)

        s = time.time()
        site.announce()
        print "Response time: %.3fs" % (time.time() - s)
        print site.peers
Esempio n. 8
0
    def siteNeedFile(self, address, inner_path):
        from Site import Site
        def checker():
            while 1:
                s = time.time()
                time.sleep(1)
                print time.time()-s
        gevent.spawn(checker)

        logging.info("Opening a simple connection server")
        global file_server
        from Connection import ConnectionServer
        file_server = ConnectionServer("127.0.0.1", 1234)

        site = Site(address)
        site.announce()
        print site.needFile(inner_path, update=True)
Esempio n. 9
0
    def siteNeedFile(self, address, inner_path):
        from Site import Site
        def checker():
            while 1:
                s = time.time()
                time.sleep(1)
                print "Switch time:", time.time()-s
        gevent.spawn(checker)

        logging.info("Opening a simple connection server")
        global file_server
        from Connection import ConnectionServer
        file_server = ConnectionServer("127.0.0.1", 1234)

        site = Site(address)
        site.announce()
        print site.needFile(inner_path, update=True)
Esempio n. 10
0
    def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", diffs={}):
        global file_server
        from Site import Site
        from Site import SiteManager
        from File import FileServer  # We need fileserver to handle incoming file requests
        from Peer import Peer
        SiteManager.site_manager.load()

        logging.info("Loading site...")
        site = Site(address, allow_create=False)
        site.settings["serving"] = True  # Serving the site even if its disabled

        logging.info("Creating FileServer....")
        file_server = FileServer()
        site.connection_server = file_server
        file_server_thread = gevent.spawn(file_server.start, check_sites=False)  # Dont check every site integrity
        time.sleep(0.001)

        if not file_server_thread.ready():
            # Started fileserver
            file_server.openport()
            if peer_ip:  # Announce ip specificed
                site.addPeer(peer_ip, peer_port)
            else:  # Just ask the tracker
                logging.info("Gathering peers from tracker")
                site.announce()  # Gather peers
            published = site.publish(5, inner_path, diffs=diffs)  # Push to peers
            if published > 0:
                time.sleep(3)
                logging.info("Serving files (max 60s)...")
                gevent.joinall([file_server_thread], timeout=60)
                logging.info("Done.")
            else:
                logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
        else:
            # Already running, notify local client on new content
            logging.info("Sending siteReload")
            if config.fileserver_ip == "*":
                my_peer = Peer("127.0.0.1", config.fileserver_port)
            else:
                my_peer = Peer(config.fileserver_ip, config.fileserver_port)

            logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path}))
            logging.info("Sending sitePublish")
            logging.info(my_peer.request("sitePublish", {"site": site.address, "inner_path": inner_path, "diffs": diffs}))
            logging.info("Done.")
Esempio n. 11
0
    def siteNeedFile(self, address, inner_path):
        from Site import Site
        from Site import SiteManager
        SiteManager.site_manager.load()

        def checker():
            while 1:
                s = time.time()
                time.sleep(1)
                print "Switch time:", time.time() - s
        gevent.spawn(checker)

        logging.info("Opening a simple connection server")
        global file_server
        from File import FileServer
        file_server = FileServer("127.0.0.1", 1234)
        file_server_thread = gevent.spawn(file_server.start, check_sites=False)

        site = Site(address)
        site.announce()
        print site.needFile(inner_path, update=True)
Esempio n. 12
0
def site_temp(request):
    with mock.patch("Config.config.data_dir", config.data_dir + "-temp"):
        site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site_temp.storage.deleteFiles()
        site_temp.content_manager.contents.db.deleteSite("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.content_manager.contents.db.close()
        db_path = "%s-temp/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]
    request.addfinalizer(cleanup)
    return site_temp
Esempio n. 13
0
def site_temp(request):
    threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)]
    with mock.patch("Config.config.data_dir", config.data_dir + "-temp"):
        site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site_temp.storage.deleteFiles()
        site_temp.content_manager.contents.db.deleteSite(site_temp)
        site_temp.content_manager.contents.db.close()
        db_path = "%s-temp/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]
        gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before])
    request.addfinalizer(cleanup)
    return site_temp
Esempio n. 14
0
def site_temp(request):
    threads_before = [obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet)]
    with mock.patch("Config.config.data_dir", config.data_dir + "-temp"):
        site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.announce = mock.MagicMock(return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site_temp.storage.deleteFiles()
        site_temp.content_manager.contents.db.deleteSite(site_temp)
        site_temp.content_manager.contents.db.close()
        db_path = "%s-temp/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]
        gevent.killall([obj for obj in gc.get_objects() if isinstance(obj, gevent.Greenlet) and obj not in threads_before])
    request.addfinalizer(cleanup)
    return site_temp
Esempio n. 15
0
def site_temp(request):
    with mock.patch("Config.config.data_dir", config.data_dir + "-temp"):
        site_temp = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.announce = mock.MagicMock(
            return_value=True)  # Don't try to find peers from the net

    def cleanup():
        site_temp.storage.deleteFiles()
        site_temp.content_manager.contents.db.deleteSite(
            "1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
        site_temp.content_manager.contents.db.close()
        db_path = "%s-temp/content.db" % config.data_dir
        os.unlink(db_path)
        del ContentDb.content_dbs[db_path]

    request.addfinalizer(cleanup)
    return site_temp
Esempio n. 16
0
def siteNeedFile(address, inner_path):
    from Site import Site
    site = Site(address)
    site.announce()
    print site.needFile(inner_path, update=True)
Esempio n. 17
0
 def siteNeedFile(self, address, inner_path):
     from Site import Site
     site = Site(address)
     site.announce()
     print site.needFile(inner_path, update=True)