class ServerHandler:
DEFAULT_PARTITIONS = 8
def __init__(self):
self.total_partitions = 0
self.task_id = 0
self.partition_id = 4
self.task_progress = {}
self.result = {}
self.scheduler = Scheduler()
self.merger = {}
def hand_shake(self, paritions):
#self.total_partitions += paritions
print("Slave Connected")
def schedule_task(self, df, func, partitions, merger):
self.task_id += 1
print("Task %d Scheduled with partitions preference = %d" %
(self.task_id, partitions))
df, func, merger = deserialize(df), deserialize(func), deserialize(
merger)
task = {"df": df, "func": func, "priority": 0, "task_id": self.task_id}
self.task_progress.setdefault(task["task_id"], {
"total": 0,
"progress": 0
})
splitted_task = partition.split_task(task, partitions)
self.task_progress[task["task_id"]]["total"] += len(splitted_task)
if merger:
self.merger[task["task_id"]] = merger
self.scheduler.schedule_tasks(splitted_task)
return self.task_id
def submit_result(self, partition_id, res):
print("Partition %s submitted" % partition_id)
task_id = self.scheduler.finish_task(partition_id)
if task_id != None:
res = deserialize(res)
self.task_progress[task_id]["progress"] += 1
if task_id in self.merger:
if task_id in self.result:
self.result[task_id] = self.merger[task_id](
self.result[task_id], res)
else:
self.result[task_id] = res
else:
self.result.setdefault(task_id, [])
self.result[task_id].append(res)
def offer_resources(self, partitions):
assigned_tasks = self.scheduler.select_tasks(partitions)
return serialize(assigned_tasks)
def ready(self, task_id):
return self.task_progress[task_id]["total"] == self.task_progress[
task_id]["progress"]
def progress(self, task_id):
return self.task_progress[task_id][
"progress"] * 100.0 / self.task_progress[task_id]["total"]
def collect(self, task_id):
return serialize(self.result[task_id])
