def testFindOptional(self, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
file_server_full = FileServer("127.0.0.1", 1546)
site_full.connection_server = file_server_full
gevent.spawn(lambda: ConnectionServer.start(file_server_full))
time.sleep(0) # Port opening
file_server_full.sites[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer("127.0.0.1", 1546) # Add it to source server
assert site_full_peer.updateHashfield() # Update hashfield
# Init client server
site_temp.connection_server = ConnectionServer("127.0.0.1", 1545)
site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net
site_temp.addPeer("127.0.0.1", 1544) # Add source server
# Download normal files
site_temp.download(blind_includes=True).join(timeout=5)
# Download optional data/optional.txt
optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt")
assert not site_temp.storage.isFile("data/optional.txt")
assert not site.content_manager.hashfield.hasHash(
optional_file_info["sha512"]
) # Source server don't know he has the file
assert site_full_peer.hashfield.hasHash(
optional_file_info["sha512"]
) # Source full peer on source server has the file
assert site_full.content_manager.hashfield.hasHash(
optional_file_info["sha512"]
) # Source full server he has the file
with Spy.Spy(FileRequest, "route") as requests:
# Request 2 file same time
threads = []
threads.append(site_temp.needFile("data/optional.txt", blocking=False))
threads.append(
site_temp.needFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif", blocking=False
)
)
gevent.joinall(threads)
assert (
len([request for request in requests if request[0] == "findHashIds"]) == 1
) # findHashids should call only once
assert site_temp.storage.isFile("data/optional.txt")
assert site_temp.storage.isFile("data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif")
assert site_temp.storage.deleteFiles()
file_server_full.stop()
[connection.close() for connection in file_server.connections]
def sitePublish(self,
address,
peer_ip=None,
peer_port=15441,
inner_path="content.json",
diffs={}):
global file_server
from Site import Site
from File import FileServer # We need fileserver to handle incoming file requests
from Peer import Peer
logging.info("Loading site...")
site = Site(address, allow_create=False)
site.settings[
"serving"] = True # Serving the site even if its disabled
logging.info("Creating FileServer....")
file_server = FileServer()
site.connection_server = file_server
file_server_thread = gevent.spawn(
file_server.start,
check_sites=False) # Dont check every site integrity
time.sleep(0.001)
if not file_server_thread.ready():
# Started fileserver
file_server.openport()
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
published = site.publish(5, inner_path,
diffs=diffs) # Push to peers
if published > 0:
time.sleep(3)
logging.info("Serving files (max 60s)...")
gevent.joinall([file_server_thread], timeout=60)
logging.info("Done.")
else:
logging.info(
"No peers found, sitePublish command only works if you already have visitors serving your site"
)
else:
# Already running, notify local client on new content
logging.info("Sending siteReload")
my_peer = Peer("127.0.0.1", config.fileserver_port)
logging.info(
my_peer.request("siteReload", {
"site": site.address,
"inner_path": inner_path
}))
logging.info("Sending sitePublish")
logging.info(
my_peer.request("sitePublish", {
"site": site.address,
"inner_path": inner_path,
"diffs": diffs
}))
logging.info("Done.")
def testFindOptional(self, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
file_server_full = FileServer("127.0.0.1", 1546)
site_full.connection_server = file_server_full
gevent.spawn(lambda: ConnectionServer.start(file_server_full))
time.sleep(0) # Port opening
file_server_full.sites[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer("127.0.0.1",
1546) # Add it to source server
assert site_full_peer.updateHashfield() # Update hashfield
# Init client server
site_temp.connection_server = ConnectionServer("127.0.0.1", 1545)
site_temp.announce = mock.MagicMock(
return_value=True) # Don't try to find peers from the net
site_temp.addPeer("127.0.0.1", 1544) # Add source server
# Download normal files
site_temp.download(blind_includes=True).join(timeout=5)
# Download optional data/optional.txt
optional_file_info = site_temp.content_manager.getFileInfo(
"data/optional.txt")
assert not site_temp.storage.isFile("data/optional.txt")
assert not site.content_manager.hashfield.hasHash(
optional_file_info["sha512"]
) # Source server don't know he has the file
assert site_full_peer.hashfield.hasHash(
optional_file_info["sha512"]
) # Source full peer on source server has the file
assert site_full.content_manager.hashfield.hasHash(
optional_file_info["sha512"]) # Source full server he has the file
with Spy.Spy(FileRequest, "route") as requests:
site_temp.needFile("data/optional.txt")
print requests
assert site_temp.storage.deleteFiles()
file_server_full.stop()
[connection.close() for connection in file_server.connections]
def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json", diffs={}):
global file_server
from Site import Site
from Site import SiteManager
from File import FileServer # We need fileserver to handle incoming file requests
from Peer import Peer
SiteManager.site_manager.load()
logging.info("Loading site...")
site = Site(address, allow_create=False)
site.settings["serving"] = True # Serving the site even if its disabled
logging.info("Creating FileServer....")
file_server = FileServer()
site.connection_server = file_server
file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
time.sleep(0.001)
if not file_server_thread.ready():
# Started fileserver
file_server.openport()
if peer_ip: # Announce ip specificed
site.addPeer(peer_ip, peer_port)
else: # Just ask the tracker
logging.info("Gathering peers from tracker")
site.announce() # Gather peers
published = site.publish(5, inner_path, diffs=diffs) # Push to peers
if published > 0:
time.sleep(3)
logging.info("Serving files (max 60s)...")
gevent.joinall([file_server_thread], timeout=60)
logging.info("Done.")
else:
logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
else:
# Already running, notify local client on new content
logging.info("Sending siteReload")
if config.fileserver_ip == "*":
my_peer = Peer("127.0.0.1", config.fileserver_port)
else:
my_peer = Peer(config.fileserver_ip, config.fileserver_port)
logging.info(my_peer.request("siteReload", {"site": site.address, "inner_path": inner_path}))
logging.info("Sending sitePublish")
logging.info(my_peer.request("sitePublish", {"site": site.address, "inner_path": inner_path, "diffs": diffs}))
logging.info("Done.")
def testFindOptional(self, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
file_server_full = FileServer("127.0.0.1", 1546)
site_full.connection_server = file_server_full
gevent.spawn(lambda: ConnectionServer.start(file_server_full))
time.sleep(0) # Port opening
file_server_full.sites[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer("127.0.0.1", 1546) # Add it to source server
assert site_full_peer.updateHashfield() # Update hashfield
# Init client server
site_temp.connection_server = ConnectionServer("127.0.0.1", 1545)
site_temp.announce = mock.MagicMock(return_value=True) # Don't try to find peers from the net
site_temp.addPeer("127.0.0.1", 1544) # Add source server
# Download normal files
site_temp.download(blind_includes=True).join(timeout=5)
# Download optional data/optional.txt
optional_file_info = site_temp.content_manager.getFileInfo("data/optional.txt")
assert not site_temp.storage.isFile("data/optional.txt")
assert not site.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source server don't know he has the file
assert site_full_peer.hashfield.hasHash(optional_file_info["sha512"]) # Source full peer on source server has the file
assert site_full.content_manager.hashfield.hasHash(optional_file_info["sha512"]) # Source full server he has the file
with Spy.Spy(FileRequest, "route") as requests:
site_temp.needFile("data/optional.txt")
print requests
assert site_temp.storage.deleteFiles()
file_server_full.stop()
[connection.close() for connection in file_server.connections]
def testFindOptional(self, file_server, site, site_temp):
file_server.ip_incoming = {} # Reset flood protection
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
file_server_full = FileServer("127.0.0.1", 1546)
site_full.connection_server = file_server_full
gevent.spawn(lambda: ConnectionServer.start(file_server_full))
time.sleep(0.001) # Port opening
file_server_full.sites[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer("127.0.0.1",
1546) # Add it to source server
assert site_full_peer.updateHashfield() # Update hashfield
# Init client server
site_temp.connection_server = ConnectionServer("127.0.0.1", 1545)
site_temp.announce = mock.MagicMock(
return_value=True) # Don't try to find peers from the net
site_temp.addPeer("127.0.0.1", 1544) # Add source server
# Download normal files
site_temp.download(blind_includes=True).join(timeout=5)
# Download optional data/optional.txt
optional_file_info = site_temp.content_manager.getFileInfo(
"data/optional.txt")
assert not site_temp.storage.isFile("data/optional.txt")
assert not site.content_manager.hashfield.hasHash(
optional_file_info["sha512"]
) # Source server don't know he has the file
assert site_full_peer.hashfield.hasHash(
optional_file_info["sha512"]
) # Source full peer on source server has the file
assert site_full.content_manager.hashfield.hasHash(
optional_file_info["sha512"]) # Source full server he has the file
with Spy.Spy(FileRequest, "route") as requests:
# Request 2 file same time
threads = []
threads.append(
site_temp.needFile("data/optional.txt", blocking=False))
threads.append(
site_temp.needFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif",
blocking=False))
gevent.joinall(threads)
assert len([
request for request in requests if request[0] == "findHashIds"
]) == 1 # findHashids should call only once
assert site_temp.storage.isFile("data/optional.txt")
assert site_temp.storage.isFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert site_temp.storage.deleteFiles()
file_server_full.stop()
[connection.close() for connection in file_server.connections]
def testFindOptional(self, file_server, site, site_temp):
# Init source server
site.connection_server = file_server
file_server.sites[site.address] = site
# Init full source server (has optional files)
site_full = Site("1TeSTvb4w2PWE81S2rEELgmX2GCCExQGT")
file_server_full = FileServer(file_server.ip, 1546)
site_full.connection_server = file_server_full
def listen():
ConnectionServer.start(file_server_full)
ConnectionServer.listen(file_server_full)
gevent.spawn(listen)
time.sleep(0.001) # Port opening
file_server_full.sites[site_full.address] = site_full # Add site
site_full.storage.verifyFiles(quick_check=True) # Check optional files
site_full_peer = site.addPeer(file_server.ip,
1546) # Add it to source server
hashfield = site_full_peer.updateHashfield() # Update hashfield
assert len(site_full.content_manager.hashfield) == 8
assert hashfield
assert site_full.storage.isFile("data/optional.txt")
assert site_full.storage.isFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert len(site_full_peer.hashfield) == 8
# Remove hashes from source server
for hash in list(site.content_manager.hashfield):
site.content_manager.hashfield.remove(hash)
# Init client server
site_temp.connection_server = ConnectionServer(file_server.ip, 1545)
site_temp.addPeer(file_server.ip, 1544) # Add source server
# Download normal files
site_temp.log.info("Start Downloading site")
site_temp.download(blind_includes=True).join(timeout=5)
# Download optional data/optional.txt
optional_file_info = site_temp.content_manager.getFileInfo(
"data/optional.txt")
optional_file_info2 = site_temp.content_manager.getFileInfo(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert not site_temp.storage.isFile("data/optional.txt")
assert not site_temp.storage.isFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert not site.content_manager.hashfield.hasHash(
optional_file_info["sha512"]
) # Source server don't know he has the file
assert not site.content_manager.hashfield.hasHash(
optional_file_info2["sha512"]
) # Source server don't know he has the file
assert site_full_peer.hashfield.hasHash(
optional_file_info["sha512"]
) # Source full peer on source server has the file
assert site_full_peer.hashfield.hasHash(
optional_file_info2["sha512"]
) # Source full peer on source server has the file
assert site_full.content_manager.hashfield.hasHash(
optional_file_info["sha512"]) # Source full server he has the file
assert site_full.content_manager.hashfield.hasHash(
optional_file_info2["sha512"]
) # Source full server he has the file
site_temp.log.info("Request optional files")
with Spy.Spy(FileRequest, "route") as requests:
# Request 2 file same time
threads = []
threads.append(
site_temp.needFile("data/optional.txt", blocking=False))
threads.append(
site_temp.needFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif",
blocking=False))
gevent.joinall(threads)
assert len([
request for request in requests if request[1] == "findHashIds"
]) == 1 # findHashids should call only once
assert site_temp.storage.isFile("data/optional.txt")
assert site_temp.storage.isFile(
"data/users/1CjfbrbwtP8Y2QjPy12vpTATkUT7oSiPQ9/peanut-butter-jelly-time.gif"
)
assert site_temp.storage.deleteFiles()
file_server_full.stop()
[connection.close() for connection in file_server.connections]
site_full.content_manager.contents.db.close()
