def __init__(self, store: Storage.Storage):
# Keep reference to storage object
self.store = store
# Initialise the LibPeer application
self.application = Application("tug")
# SODI instance on default channel
self.sodi = SODI(self.application)
# Listen for solicitations
self.sodi.solicited.subscribe(self.handle_solicitation)
# Holds open channels
self.channels = {}
# Download channels
self.dsis = {}
# Advertise initial artefact checksums we offer
for checksum in self.store.get_artefact_checksums():
print("Added label for {}".format(Checksum.stringify(checksum)))
# Advertise with label
#self.application.add_label(checksum)
# Make this instance discoverable
self.application.set_discoverable(True)
def publish_folder(directiory):
structure = get_structure(directiory)
final = None
for artefact in Publish.Map.Create(structure):
file_system.save_artefact(Storage.STORAGE_MIRROR, artefact)
final = artefact
print("tug://{}".format(Checksum.stringify(final.checksum)))
def get_artefact(self, checksum, expected_size = 0):
# Get the artefact path
path = next(path for path in (os.path.join(x, Checksum.stringify(checksum)) for x in self.path_types.values()) if os.path.exists(path))
# Get atrefact stream
stream = open(path, "rb")
# Return artefact
return ArtefactFactory.deserialise(stream, expected_size)
def do_GET(self):
self.done = False
# Attempt to get the artefact
subject = protocol.retrieve_artefact(Checksum.parse(self.path[1:]))
subject.subscribe(self.got_artefact, self.error)
while not self.done:
pass
def handle_map(self, artefact: Map):
self.send_response(200)
self.end_headers()
html = """
<h1>Map Listing for {}</h1>
<hr/>
<ul>
""".format(Checksum.stringify(artefact.checksum))
for entry in artefact.destinations:
html += """
<li><a href="/{}">{}</a></li>
""".format(Checksum.stringify(entry.reference.checksum),
entry.name)
html += """</ul>"""
self.wfile.write(html.encode("utf-8"))
self.wfile.flush()
self.done = True
def save_artefact(self, storage_type, artefact):
# Get path for artefact
path = os.path.join(self.path_types[storage_type], Checksum.stringify(artefact.checksum))
# Get stream to save to
stream = open(path, "wb")
# Write path artefact to stream
for data in artefact.as_sendable():
stream.write(data)
stream.close()
def handle_solicitation(self, solicitation):
# Get checksum from solicitation
checksum = Checksum.parse(solicitation.query)
# Solicitation string contains hash of artefact
has_artefact = self.store.has_artefact(checksum)
# What channel at this peer to connect to in order to receive the artefact
find_at_channel = None
# Do we have the artefact?
if (has_artefact):
# Get the channel identifier
find_at_channel = self.get_channel(checksum).uuid
# Reply to the solicitation
solicitation.reply({
"has_artefact": has_artefact,
"find_at_channel": base64.b64encode(find_at_channel)
})
def get_artefact_checksums(self):
yield from (Checksum.parse(x) for x in os.listdir(self.mirror_path))
yield from (Checksum.parse(x) for x in os.listdir(self.assistive_path))
yield from (Checksum.parse(x) for x in os.listdir(self.participatory_path))
def remove_artefact(self, checksum):
# Get the artefact storage type
storage_type = self.get_storage_type(checksum)
# Remove from disk
os.remove(os.path.join(self.path_types[storage_type], Checksum.stringify(checksum)))
def get_storage_type(self, checksum):
# Get the storage type
return next(key for key, value in self.path_types.items() if os.path.exists(os.path.join(value, Checksum.stringify(checksum))))
def has_artefact(self, checksum):
# Check if we have an artefact
return any(os.path.exists(os.path.join(x, Checksum.stringify(checksum))) for x in self.path_types.values())
def retrieve_artefact(self,
checksum,
storage_policy=Storage.STORAGE_PARTICIPATORY,
peers_to_try=10):
# TODO Remove
print("Retreive {}".format(Checksum.stringify(checksum)))
# Create the subject to give to the caller
subject = rx.subjects.ReplaySubject()
# Do we already have the artefact in storage?
if (self.store.has_artefact(checksum)):
# Return that instead
try:
subject.on_next(self.store.get_artefact(checksum))
except Exception as e:
subject.on_error(e)
return subject
# Find peers claiming to have the artefact
#discoveries = self.application.find_peers_with_label(checksum)
discoveries = self.application.find_peers()
if (len(discoveries) == 0):
# If there were no discoveries, return the subject with an error
subject.on_error(
Exception(
"No peers found advertising themselves as having the requested artefact"
))
return subject
# Pick random peers to solicit
random.shuffle(discoveries)
discoveries = discoveries[:peers_to_try]
got_response = False
responses = 0
def handle_reply(reply):
nonlocal responses
nonlocal got_response
nonlocal subject
nonlocal storage_policy
# Increment responses
responses += 1
try:
# Get object from reply
obj = reply.get_object()
# Peer replied after another peer or doesn't have the object
if (got_response or not obj["has_artefact"]):
# Have we had the number of responses we sent?
if (responses == peers_to_try):
subject.on_error(
Exception(
"All peers tried could not deliver artefact"))
# Stop execution
return
# We have our response
got_response = True
def get_it():
nonlocal subject
nonlocal storage_policy
# Get a DSI to get the artefact from
dsi = self.get_dsi(base64.b64decode(
obj["find_at_channel"]))
# Connect to the peer at this channel
connection = dsi.connect(reply.peer)
# Create an artefact
artefact = ArtefactFactory.deserialise(connection)
# Save the artefact
self.store.save_artefact(storage_policy, artefact)
# Return the artefact to the subject
subject.on_next(artefact)
threading.Thread(target=get_it).start()
except Exception as e:
subject.on_error(e)
def handle_error(exception):
nonlocal responses
nonlocal got_response
nonlocal subject
# Increment responses
responses += 1
# Have we had the number of responses we sent?
if (responses == peers_to_try):
subject.on_error(
Exception("All peers tried could not deliver artefact"))
# Loop over each peer
for discovery in discoveries:
# Solicit the peer
self.sodi.solicit(discovery.peer,
Checksum.stringify(checksum)).subscribe(
handle_reply, handle_error)
# Return the subject
return subject