def request(self, args):
client, sizehint = args[:2]
self.update_files() # TODO: fix the race condition behind this
filelist = [ filename for filename in self.scanner.keys() \
if client not in self.files[filename] \
and not self.held(filename, client) ]
filelist = sorted(filelist, key=lambda f: self.scanner[f]["size"])
filelist = sorted(filelist, key=lambda f: len(self.files[f]))
self.logger.debug(f"pulled a list for {client}:")
self.dump_files(filelist)
if not filelist: # no files for this client
return Communique("__none__", negatives=("__none__", ))
target = len(self.files[filelist[0]]) + 1
if target < self.copies: # implies underserved; expand to
target = self.copies # consider any underserved file
candidates = [ filename for filename in filelist \
if len(self.files[filename]) < target ]
self.logger.debug(f"targeted list for {client}:")
self.dump_files(candidates)
# TODO: return multiple files
filename = self.least_served(candidates, int(sizehint))
self.logger.debug(f"least_served gives {filename}")
if filename:
self.hold(filename, client)
return Communique(filename, str(self.scanner[filename]["size"]))
else:
return Communique("__none__", negatives=("__none__", ))
def claim(self, args):
client, filename, checksum = args[:3]
if filename not in self.scanner:
self.logger.warn("Client has a file, I don't; deleted?")
return Communique("drop", truthiness=False)
return Communique("NACK", truthiness=False)
filestate = self.scanner[filename]
self.logger.debug(f"{client} claims file {filename}")
if filestate["checksum"] == "deferred":
self.logger.debug(
"I have a deferred checksum; let it go (for now)")
# return Communique("keep", truthiness=True)
# fallthrough
elif checksum != filestate["checksum"]:
self.logger.warn(f"{client} has a different checksum ...")
self.scanner.update(filename)
filestate = self.scanner[filename]
if checksum != filestate["checksum"]:
self.logger.warn(f"{client} has the wrong checksum!\n" * 10)
return Communique("update", truthiness=False)
else:
self.logger.warn(f"{client} was right; I'm straight now")
return Communique("keep", truthiness=True)
if filename not in self.files:
self.logger.warn(f"I'm learning about {filename}")
self.files[filename] = [client]
elif client not in self.files[filename]:
self.files[filename].append(client)
self.stats['claims'] += 1
self.release(filename)
# return Communique("keep", truthiness=True)
return Communique("ack")
def status(self, args):
client = args[0]
response = []
if self.underserved(args):
response.append("underserved")
if self.request((client, 0)):
response.append("available")
if self.overserved(args):
response.append("overserved")
if response:
return Communique(response)
else:
return Communique("just right")
def unclaim(self, args):
client, filename = args[:2]
# self.logger.debug(f"files: {self.files}, client: {client}, filename: {filename}")
if filename and len(self.files) > 0 and filename in self.files \
and client in self.files[filename]:
self.files[filename].remove(client)
self.stats['drops'] += 1
return Communique("ack")
def overserved(self, args):
client = args[0]
files = {}
for filename in self.files:
# self.logger.debug(f"{filename}: {len(self.files[filename])}/{self.copies} {self.files[filename]}")
if client in self.files[filename]:
# self.logger.debug(f"client match for {filename}")
if len(self.files[filename]) > self.copies:
# self.logger.debug(f"overserved file: {filename}")
files[filename] = len(self.files[filename])
if len(files.keys()) > 0:
filenames = sorted(files.keys(),
key=lambda x: files[x],
reverse=True)
return Communique(self.random_subset(filenames, 20))
else:
self.logger.debug("no overserved files :/")
return Communique(None)
def handle(self, data):
request = Communique.build(data)
action, server_context = request[:2]
args = request[2:]
if server_context not in self.servlets:
return "__none__"
# self.logger.debug(f"acting: {server_context} => {action}({args})")
response = self.servlets[server_context].handle(action, args)
if not response:
return "__none__"
return str(response)
def inventory(self, args):
client = args[0]
self.logger.debug(f"{client} wants inventory")
files = []
for filename in self.files:
if client in self.files[filename]:
files.append(filename)
# else:
# self.logger.debug(f"{client} not in {filename}")
# TODO: make this better
c = Communique(files)
self.logger.debug(f"inventory: {c}")
return c
def underserved(self, args):
client = args[0]
self.logger.debug(f"Underserved for {client}?")
# self.audit()
files = []
for filename in self.files:
if client not in self.files[filename] and \
len(self.files[filename]) < self.copies:
# weighted random... TODO make this ~O(1)
files.append(filename)
# return filename
if len(files) > 0:
# TODO: return a subset of the list
return self.random_subset(files, 20)
# file = random.choice(files)
# self.logger.debug(f"returning {file} and moar")
return files
self.logger.debug("I got nothin'")
return Communique(None)
def heartbeep(self, args):
client = args[0]
self.logger.info(f"heartbeep from {client}: {args}")
return Communique("ack")
def drain(self, args):
client, filename = args[:2]
self.logger.debug(f"client: {client} requests drain {filename}")
self.drains[f"{client}:{filename}"] = 1
return Communique("ack")