def monitor(self, worker): try: rpc.getThriftWorkerManagerClient('workermanager').update_status(worker.instance_name) return True except NebuException, e: self.nebu_e = e return False
def monitor(self, worker): try: rpc.getThriftWorkerManagerClient('workermanager').update_status( worker.instance_name) return True except NebuException, e: self.nebu_e = e return False
def __find_best_worker(self, required_ram): ''' finds the 'best' worker to use for a deploy requiring 'required_ram' constraints: * the worker can accomodate 'required_ram' * the worker is not in 'decomissioning' state for all those workers that comply to the above constraint, pick one considering: * workers in 'controllable' state are better than other workers * to choose between 2 workers, having less ram available is better if there is NO worker matching the constraints, this method will try to create a new worker ''' # make sure at least 1 worker matches the constraints workers = Worker.objects.exclude(status=Worker.States.decommissioning).exclude(status=Worker.States.dying).exclude(status=Worker.States.dead) should_create_worker = True for worker in workers: used = self.__get_total_ram(worker) available = worker.get_usable_ram() - used if available > required_ram: # found one that could accomodate 'required_ram' .. no need to look further. should_create_worker = False break if should_create_worker: # create a new worker. it will be available on next loop iteration wm_client = rpc.getThriftWorkerManagerClient('workermanager') wm_client.add_worker('m2.xlarge') # OK, either there were workers that could accomodate 'required_ram', or we just created one. # find the best # first try only controllable workers # this way the deploy will be available FASTER workers = Worker.objects.filter(status=Worker.States.controllable) for i in range(2): best_worker = None best_ram = 1000000 # we should bump this number when we get workers with ram > 1TB for worker in workers: used = self.__get_total_ram(worker) available = worker.get_usable_ram() - used if available > required_ram and available < best_ram: best_ram = available best_worker = worker if best_worker is not None: return best_worker # need to try again. let workers be everything but decomissioning, dying or dead ones. workers = Worker.objects.exclude(status=Worker.States.decommissioning).exclude(status=Worker.States.dying).exclude(status=Worker.States.dead) # tried only controllable, and everything .. still not found raise Exception('Second iteration and no suitable worker. Something is wrong...')
from rpc import getThriftWorkerManagerClient import sys # Missing a way to close transport if __name__ == '__main__': client = getThriftWorkerManagerClient('workermanager') if len(sys.argv) > 1: itype = sys.argv[1] retcode = client.add_worker(itype) print "Finished adding worker of type [%s] : %s " % (itype, retcode) else: retcode = client.add_worker() print "Finished adding worker: %s" % retcode