def check_for_changes_test(self): print "***** Testing check_for_changes *****" self.reset_stuff() #now make a new entry into database to have a only one #new entry in the db ... #running a new command which is a short one new_fc = Overlord("*",async=True) new_job_id=new_fc.test.add(1,2) #print "The job id we got is :",new_job_id changes = self.async_manager.check_for_changes() print "The latest Changes for add method are :",changes assert len(changes) == 1 assert changes[0] == new_job_id #check if that one is finished another_test = False while new_fc.job_status(new_job_id)[0] != JOB_ID_FINISHED: print "Waiting for add command to finish " time.sleep(2) another_test = True # that probably may happen so should add it here if another_test: changes = self.async_manager.check_for_changes() assert len(changes) == 1 assert changes[0] == new_job_id print "The changes are for add finish :",changes #now should run another command that is longer to see what happens new_job_id = new_fc.test.sleep(4) # we have now one entry in the db what to do ? # when now run the check changes should have ne entry in the changes :) changes = self.async_manager.check_for_changes() print "The changes for sleep are :",changes assert len(changes) == 1 assert changes[0] == new_job_id #if we already have the finished message we dont have to run the other test after that one another_test = False while new_fc.job_status(new_job_id)[0] != JOB_ID_FINISHED: print "Waiting for sleep command to finish " time.sleep(2) another_test = True if another_test: changes = self.async_manager.check_for_changes() assert len(changes) == 1 assert changes[0] == new_job_id print "The changes for sleep finish are :",changes
def check_job_status(self, job_id): """ Checking the job status for specific job_id that method will be useful to see the results from async_results table ... """ if not job_id: return dict(result="job id shouldn be empty!") if not self.func_cache["fc_async_obj"]: if self.func_cache["glob"]: fc_async = Overlord(self.func_cache["glob"], async=True) # store also into the cache else: fc_async = Overlord("*", async=True) self.func_cache["fc_async_obj"] = fc_async else: fc_async = self.func_cache["fc_async_obj"] id_result = fc_async.job_status(job_id) # the final id_result return dict(result=id_result)
def upgrade_policy(self): """ Update the SELinux policy across the given minions """ print "Cleaning yum metadata..." results = self.command.run('yum clean metadata') for minion, result in results.items(): if result[0]: print "[%s] Problem cleaning yum cache: %s" % (minion, result[1]) async_client = Overlord(self.minion_glob, nforks=10, async=True) print "Upgrading SELinux policy..." job_id = async_client.command.run('yum -y update selinux\*') running = True while running: time.sleep(20) return_code, results = async_client.job_status(job_id) if return_code in (jobthing.JOB_ID_RUNNING, jobthing.JOB_ID_PARTIAL): continue elif return_code == jobthing.JOB_ID_FINISHED: for minion, result in results.items(): if result[0]: print '[%s] Problem upgrading policy: %s' % (minion, result[1]) if 'Updated: selinux-policy' in result[1]: ver = result[1].split( 'Updated: ')[-1].split()[1].split(':')[1] print "[%s] selinux-policy successfully upgraded to %s" % ( minion, ver) else: print "selinux-policy *not* upgraded on %s: %s" % ( minion, result[1]) running = False elif return_code == jobthing.JOB_ID_LOST_IN_SPACE: print "Job %s lost in space: %s" % (job_id, results) else: print "Unknown return code %s: %s" % (return_code, results) print "SELinux policy upgrade complete!"
def upgrade_policy(self): """ Update the SELinux policy across the given minions """ print "Cleaning yum metadata..." results = self.command.run('yum clean metadata') for minion, result in results.items(): if result[0]: print "[%s] Problem cleaning yum cache: %s" % \ (minion, result[1]) async_client = Overlord(self.minion_glob, nforks=10, async=True) print "Upgrading SELinux policy..." job_id = async_client.command.run('yum -y update selinux\*') running = True while running: time.sleep(20) return_code, results = async_client.job_status(job_id) if return_code in (jobthing.JOB_ID_RUNNING, jobthing.JOB_ID_PARTIAL): continue elif return_code == jobthing.JOB_ID_FINISHED: for minion, result in results.items(): if result[0]: print '[%s] Problem upgrading policy: %s' % \ (minion, result[1]) if 'Updated: selinux-policy' in result[1]: ver = result[1].split('Updated: ')[-1].split()[1]. \ split(':')[1] print "[%s] selinux-policy successfully upgraded to " \ "%s" % (minion, ver) else: print "selinux-policy *not* upgraded on %s: %s" % \ (minion, result[1]) running = False elif return_code == jobthing.JOB_ID_LOST_IN_SPACE: print "Job %s lost in space: %s" % (job_id, results) else: print "Unknown return code %s: %s" % (return_code, results) print "SELinux policy upgrade complete!"
class AsyncResultManager(object): """ A class to check the async result updates,changes to be able to display on UI """ JOB_CODE_CHANGED = 1 JOB_CODE_NEW = 2 JOB_CODE_SAME = 3 pull_property_options = ('FINISHED','ERROR','NEW','CHANGED','RUNNING','PARTIAL') def __init__(self): #we keep all the entries in memory it may seems #that it will occuppy lots of space but all it has #is job_id:code pairs with some status info : changed,new #and etc all other stuff is kept in DB #so if someone does 1000 async queries will occupy #1000 * (integer*2) not big deal :) #the format will be job_id : [code,status] self.__current_list = {} #create a dummy Overlord object self.fc = Overlord("*") def __get_current_list(self,check_for_change=False): """ Method returns back the current list of the job_ids : result_code pairs """ changed = [] tmp_ids = self.fc.open_job_ids() for job_id,code in tmp_ids.iteritems(): #is it a new code ? if self.__current_list.has_key(job_id): #the code is same no change occured if self.__current_list[job_id][0] == code: #print "I have that code %s no change will be reported"%job_id self.__current_list[job_id][1] = self.JOB_CODE_SAME else: #we have change i db #print "That is a change from %d to %d for %s"%(self.__current_list[job_id][0],code,job_id) self.__current_list[job_id]=[code,self.JOB_CODE_CHANGED] if check_for_change: changed.append(job_id) else: # a new code was added #print "A new code was added %s"%job_id self.__current_list[job_id] = [code,self.JOB_CODE_NEW] if check_for_change: changed.append(job_id) #if true the db was updated and ours is outofdate if len(self.__current_list.keys()) != len(tmp_ids.keys()): self.__update_current_list(tmp_ids.keys()) #if we want to know if sth has changed if check_for_change and changed: return changed return None def __update_current_list(self,tmp_db_hash): """ Synch the memory and local db """ for mem_job_id in self.__current_list.keys(): if mem_job_id not in tmp_db_hash: del self.__current_list[mem_job_id] def check_for_changes(self): """ Method will be called by js on client side to check if something interesting happened in db in "before defined" time interval If have lots of methods running async that may take a while to finish but user will not be interrupted about that situation ... """ tmp_ids = self.fc.open_job_ids() should_check_change = False for job_id,code in tmp_ids.iteritems(): #check only the partials and others if code == JOB_ID_RUNNING or code == JOB_ID_PARTIAL: #that operation updates the db at the same time try : #print "The status from %s is %s in check_for_changes"%(job_id,self.fc.job_status(job_id)[0]) tmp_code = self.fc.job_status(job_id)[0] #should_check_change = True except Exception,e: print "Some exception in pulling the job_id_status",e continue #else: # print "The job_id is not checked remotely :%s in check_for_changes and the code is %s"%(job_id,code) #if you thing there is sth to check interesting send it #if should_check_change: return self.__get_current_list(check_for_change=True)
from func.overlord.client import Overlord from func.jobthing import * import time print "Now running one with getattr " module = "echo" method = "run_int" fc_new = Overlord("*",async = True) new_job_id = getattr(getattr(fc_new,module),method)(500) code_status = fc_new.job_status(new_job_id)[0] print "The code status is : ",code_status while code_status != JOB_ID_FINISHED: print "Waiting the run_int to finish " code_status = fc_new.job_status(new_job_id)[0] time.sleep(2) print "The int operation is finished" print "Creating the object" fc = Overlord("*",async = True) job_id = fc.echo.run_string("Merhaba") code_status = fc.job_status(job_id)[0] print "The code status is : ",code_status while code_status != JOB_ID_FINISHED:
class AsyncResultManager(object): """ A class to check the async result updates,changes to be able to display on UI """ JOB_CODE_CHANGED = 1 JOB_CODE_NEW = 2 JOB_CODE_SAME = 3 pull_property_options = ('FINISHED', 'ERROR', 'NEW', 'CHANGED', 'RUNNING', 'PARTIAL') def __init__(self): #we keep all the entries in memory it may seems #that it will occuppy lots of space but all it has #is job_id:code pairs with some status info : changed,new #and etc all other stuff is kept in DB #so if someone does 1000 async queries will occupy #1000 * (integer*2) not big deal :) #the format will be job_id : [code,status] self.__current_list = {} #create a dummy Overlord object self.fc = Overlord("*") def __get_current_list(self, check_for_change=False): """ Method returns back the current list of the job_ids : result_code pairs """ changed = [] tmp_ids = self.fc.open_job_ids() for job_id, code in tmp_ids.iteritems(): #is it a new code ? if self.__current_list.has_key(job_id): #the code is same no change occured if self.__current_list[job_id][0] == code: #print "I have that code %s no change will be reported"%job_id self.__current_list[job_id][1] = self.JOB_CODE_SAME else: #we have change i db #print "That is a change from %d to %d for %s"%(self.__current_list[job_id][0],code,job_id) self.__current_list[job_id] = [code, self.JOB_CODE_CHANGED] if check_for_change: changed.append(job_id) else: # a new code was added #print "A new code was added %s"%job_id self.__current_list[job_id] = [code, self.JOB_CODE_NEW] if check_for_change: changed.append(job_id) #if true the db was updated and ours is outofdate if len(self.__current_list.keys()) != len(tmp_ids.keys()): self.__update_current_list(tmp_ids.keys()) #if we want to know if sth has changed if check_for_change and changed: return changed return None def __update_current_list(self, tmp_db_hash): """ Synch the memory and local db """ for mem_job_id in self.__current_list.keys(): if mem_job_id not in tmp_db_hash: del self.__current_list[mem_job_id] def check_for_changes(self): """ Method will be called by js on client side to check if something interesting happened in db in "before defined" time interval If have lots of methods running async that may take a while to finish but user will not be interrupted about that situation ... """ tmp_ids = self.fc.open_job_ids() should_check_change = False for job_id, code in tmp_ids.iteritems(): #check only the partials and others if code == JOB_ID_RUNNING or code == JOB_ID_PARTIAL: #that operation updates the db at the same time try: #print "The status from %s is %s in check_for_changes"%(job_id,self.fc.job_status(job_id)[0]) tmp_code = self.fc.job_status(job_id)[0] #should_check_change = True except Exception, e: print "Some exception in pulling the job_id_status", e continue #else: # print "The job_id is not checked remotely :%s in check_for_changes and the code is %s"%(job_id,code) #if you thing there is sth to check interesting send it #if should_check_change: return self.__get_current_list(check_for_change=True)
print "TESTING command that sleeps %s seconds" % TEST_SLEEP job_id = overlord.test.sleep(TEST_SLEEP) elif test == QUICK_COMMAND: print "TESTING a quick command" job_id = overlord.test.add(1,2) elif test == RAISES_EXCEPTION_COMMAND: print "TESTING a command that deliberately raises an exception" job_id = overlord.test.explode() # doesn't work yet elif test == FAKE_COMMAND: print "TESTING a command that does not exist" job_id = overlord.test.does_not_exist(1,2) # ditto print "======================================================" print "job_id = %s" % job_id while True: status = overlord.job_status(job_id) (code, results) = status nowtime = time.time() delta = int(nowtime - oldtime) if nowtime > oldtime + TEST_SLEEP + EXTRA_SLEEP: print "time expired, test failed" return if code == jobthing.JOB_ID_RUNNING: print "task is still running, %s elapsed ..." % delta elif code == jobthing.JOB_ID_ASYNC_PARTIAL: print "task reports partial status, %s elapsed, results = %s" % (delta, results) elif code == jobthing.JOB_ID_FINISHED: print "(non-async) task complete, %s elapsed, results = %s" % (delta, results) return elif code == jobthing.JOB_ID_ASYNC_FINISHED: print "(async) task complete, %s elapsed, results = %s" % (delta, results)
# print "TESTING a quick command" job_id = overlord.test.ping() # job_id = overlord.test.add(1,2) elif test == RAISES_EXCEPTION_COMMAND: print "TESTING a command that deliberately raises an exception" job_id = overlord.test.explode() # doesn't work yet elif test == FAKE_COMMAND: print "TESTING a command that does not exist" job_id = overlord.test.does_not_exist(1, 2) # ditto # print "======================================================" print "job_id = %s" % job_id while True: status = overlord.job_status(job_id) print "job_status: %s" % status[0] (code, results) = status nowtime = time.time() delta = int(nowtime - oldtime) if nowtime > oldtime + TEST_SLEEP + EXTRA_SLEEP: print "time expired, test failed" return if code == jobthing.JOB_ID_RUNNING: print "task is still running, %s elapsed ..." % delta elif code == jobthing.JOB_ID_PARTIAL: print "task reports partial status, %s elapsed, results = %s" % ( delta, results) elif code == jobthing.JOB_ID_FINISHED: print "task complete, %s elapsed, results = %s" % (delta, results)
def select_from_test(self): print "****Testing select_from**** " #these tests are a little bit tricky so may not have #the exact results all depends on remote machines :) self.reset_stuff() new_fc = Overlord("*",async=True) #new_job_id=new_fc.test.add(1,2) new_job_id = new_fc.test.sleep(6) #insert one running #now we have one entry into async_manager result_ids = new_fc.open_job_ids() self.async_manager.refresh_list() if result_ids.has_key(new_job_id) and result_ids[new_job_id] == JOB_ID_RUNNING: print "Testing for SELECT RUNNING ..." select_list = self.async_manager.select_from('RUNNING') #print "Result form selct RUNNING :",select_list assert len(select_list) == 1 assert select_list[0].has_key(new_job_id) assert select_list[0][new_job_id][0] == JOB_ID_RUNNING #pull_property_options = ('FINISHED','ERROR','NEW','CHANGED','RUNNING','PARTIAL') #insert one that finishes #get one NEW print "Testing for SELECT NEW ..." select_list = self.async_manager.select_from('NEW') #print "The select list is :",select_list assert len(select_list) == 1 assert select_list[0].has_key(new_job_id) assert select_list[0][new_job_id][1] == self.async_manager.JOB_CODE_NEW #test the ones that are changed :) another_test = False current_job_status = new_fc.job_status(new_job_id)[0] while current_job_status != JOB_ID_FINISHED: print "Waiting for sleep command to finish " time.sleep(1) another_test = True #test also for partial resultst status if current_job_status == JOB_ID_PARTIAL: #populate the list print "Testing for SELECT PARTIAL ..." self.async_manager.refresh_list() select_list = self.async_manager.select_from('PARTIAL') assert select_list[0].has_key(new_job_id) assert select_list[0][new_job_id][0] == JOB_ID_PARTIAL current_job_status = new_fc.job_status(new_job_id)[0] if another_test: print "Testing for SELECT CHANGED ..." self.async_manager.refresh_list() select_list = self.async_manager.select_from('CHANGED') #print "current Select list is :",select_list assert len(select_list) == 1 assert select_list[0].has_key(new_job_id) assert select_list[0][new_job_id][1] == self.async_manager.JOB_CODE_CHANGED print "Testing for SELECT FINISHED ..." assert select_list[0][new_job_id][0] == JOB_ID_FINISHED
def post_form(self, **kw): """ Data processing part for methods that accept some inputs. Method recieves the method arguments for minion method then orders them into their original order and sends the xmlrpc request to the minion ! """ if kw.has_key("minion") and kw.has_key("module") and kw.has_key("method"): # assign them because we need the rest so dont control everytime # and dont make lookup everytime ... # the del statements above are important dont remove them :) minion = kw["minion"] del kw["minion"] module = kw["module"] del kw["module"] method = kw["method"] del kw["method"] if self.func_cache["minion_name"] == minion: fc = self.func_cache["fc_object"] else: fc = Overlord(minion) self.func_cache["fc_object"] = fc self.func_cache["minion_name"] = minion # reset the children :) self.func_cache["module_name"] = module self.func_cache["modules"] = None self.func_cache["methods"] = None # get again the method args to get their order : arguments = getattr(fc, module).get_method_args() # so we know the order just allocate and put them there cmd_args = [""] * (len(kw.keys())) for arg in kw.keys(): # wow what a lookup :) index_of_arg = arguments[minion][method]["args"][arg]["order"] cmd_args[index_of_arg] = kw[arg] # now execute the stuff # at the final execute it as a multiple if the glob suits for that # if not (actually there shouldnt be an option like that but who knows :)) # it will run as a normal single command to clicked minion if self.func_cache["glob"]: fc_async = Overlord(self.func_cache["glob"], async=True) result_id = getattr(getattr(fc_async, module), method)(*cmd_args) result = "".join( [ "The id for current job is :", str(result_id), " You will be notified when there is some change about that command !", ] ) # that part gives a chance for short methods to finish their jobs and display them # immediately so user will not wait for new notifications for that short thing import time time.sleep(4) tmp_as_res = fc_async.job_status(result_id) if tmp_as_res[0] == JOB_ID_FINISHED: result = tmp_as_res[1] if not self.async_manager: # cleanup tha database firstly purge_old_jobs() self.async_manager = AsyncResultManager() self.async_manager.refresh_list() # TODO reformat that returning string to be more elegant to display :) return str(result) else: return "Missing arguments sorry can not proceess the form"