def start(self,port,host=None): #create heartbeat thread self.hb_thread=HeartBeatManager(self) self.db_cleaner=DatabaseCleaner() try: @Serve(port,host,[self.register,self.execute_on_grid,self.send_heartbeat]) def __start(): logger.info('Starting server on %s'%(port)) #start the heart beat manager self.hb_thread.start() self.db_cleaner.start() __start() except KeyboardInterrupt,SystemExit: #stop heartbeat manager self.hb_thread.stop.set() self.db_cleaner.stop.set() #hb_thread.join() #db_cleaner.join() logger.info('Shutting down') raise
class Master(object): def __init__(self): #This will be used to keep track of all the available nodes #This list will contain ServerProxy objects self.nodes=deque([]) #This will be used to control how many threads can be started #at once to handle requests self.tokens=deque(range(MAX_THREADS)) #This dict is used to keep track of all the heartbeats received #from the nodes - if the value in this is greater than the max #delay then the proxy is removed from the nodes list self.heartbeat={} #heartbeat thread self.hb_thread=None #db cleaner self.db_cleaner=None """ register - This method will be exposed over the XML RPC interface to allow nodes to register with this master server. This takes 2 parameters - the fully qualified domain name (FQDN) of the node and the port where the node is running. """ def register(self,ip,port): #create the URL url="http://%s:%s"%(ip,port) #append this to the list self.nodes.append(url) """ execute_on_grid - This method will be exposed over the XML RPC interface to allow the clients to submit jobs to be executed on the grid. This takes the job ID, serialized code and the arguments and send it to an available node in a separate thread. """ def execute_on_grid(self,job_id,code,args): #check and init the state DB db_mgr=DbManager() db_mgr.init_db() #getthe proxy - this will also block url=self.__getProxy() #get the token - the method will block till we get a token token=self.__getToken() #create an inline method decorated with the JobRunner and call it #it is a dummy method @JobRunner(job_id,code,args,db_mgr,token,url) def __wrap(): pass #release the proxy so that new jobs can be scheduled on this self.nodes.append(url) #return the result res=__wrap #release the token self.tokens.append(token) return res """ send_heartbeat - This records the heartbeat from a node """ def send_heartbeat(self,ip,port): #create URL and record time url="http://%s:%s"%(ip,port) self.heartbeat[url]=time.time() """ start- This starts the master using the given port. """ def start(self,port,host=None): #create heartbeat thread self.hb_thread=HeartBeatManager(self) self.db_cleaner=DatabaseCleaner() try: @Serve(port,host,[self.register,self.execute_on_grid,self.send_heartbeat]) def __start(): logger.info('Starting server on %s'%(port)) #start the heart beat manager self.hb_thread.start() self.db_cleaner.start() __start() except KeyboardInterrupt,SystemExit: #stop heartbeat manager self.hb_thread.stop.set() self.db_cleaner.stop.set() #hb_thread.join() #db_cleaner.join() logger.info('Shutting down') raise