def fetchSelected(self, idList): model = PluginTable() count = 0 for id in idList: try: data = self.fetchPlugin(int(id)) if not data: print "plugin not exist ", id continue model.save(data) count+=1 print "fetch plugin success ", id except: print "fetch plugin error ", id traceback.print_exc() continue return count
def fetchAll(self, start=1, end=0): if end <= 0: end = self.getTotal() if end < start: return False pool=0 pool_size=20 model = PluginTable() count = end - start for id in xrange(start, end): try: if pool <= pool_size: pid=os.fork() else: time.sleep(pool-pool_size) pid=os.fork() except: print "fork error" traceback.print_exc() os._exit(-1) if pid: #parent pool=pool+1 #print "fork pool=%d ", pid, id while True: result=os.waitpid(-1, os.WNOHANG) if not result[0]: break; pool=pool-1 continue else: #child print "fetch script", id try: data=self.fetchPlugin(id) if data: #print "save script", id model.save(data) os._exit(0) else: print "cannot get script", id os._exit(-2) except: print "error in getting script", id traceback.print_exc() os._exit(-1) time.sleep(3) while True: try: result = os.waitpid(-1, os.WNOHANG) if not result[0]: print "Task finished" break if result[1] < 0: count-=1 except: break; return count