def decodeJson(self, result): """ decodeJson decode the response result reveiced from the server """ encoder = JSONRequests() return encoder.decode(result)
def decodeJson(self, result): """ decodeJson decode the response result reveiced from the server """ encoder = JSONRequests(idict={"pycurl" : True}) return encoder.decode(result)
def decodeJson(self, result): """ decodeJson decode the response result reveiced from the server """ encoder = JSONRequests(idict={"pycurl": True}) return encoder.decode(result)
def decodeJson(self, result): """ decodeJson decode the response result reveiced from the server """ encoder = JSONRequests() return encoder.decode(result)
class ProcessPool: def __init__(self, slaveClassName, totalSlaves, componentDir, config, namespace = 'WMComponent', inPort = '5555', outPort = '5558'): """ __init__ Constructor for the process pool. The slave class name must be based inside the WMComponent namespace. For examples, the JobAccountant would pass in 'JobAccountant.AccountantWorker' to run the AccountantWorker class. All log files will be stored in the component directory that is passed in. Each slave will have its own log file. Note that the config is only used to determine database connection parameters. It is not passed to the slave class. The slaveInit parameter will be serialized and passed to the slave class's constructor. """ self.enqueueIndex = 0 self.dequeueIndex = 0 self.runningWork = 0 #Use the Services.Requests JSONizer, which handles __to_json__ calls self.jsonHandler = JSONRequests() # heartbeat should be registered at this point if getattr(config.Agent, "useHeartbeat", True): self.heartbeatAPI = HeartbeatAPI(getattr(config.Agent, "componentName", "ProcPoolSlave")) self.slaveClassName = slaveClassName self.componentDir = componentDir self.config = config # Grab the python version from the current version # Assume naming convention pythonA.B, i.e., python2.4 for v2.4.X majorVersion = sys.version_info[0] minorVersion = sys.version_info[1] if majorVersion and minorVersion: self.versionString = "python%i.%i" % (majorVersion, minorVersion) else: self.versionString = "python2.6" self.workers = [] self.nSlaves = totalSlaves self.namespace = namespace self.inPort = inPort self.outPort = outPort # Pickle the config self.configPath = os.path.join(componentDir, '%s_config.pkl' % slaveClassName) if os.path.exists(self.configPath): # Then we note it and overwrite it msg = "Something's in the way of the ProcessPool config: %s" % self.configPath logging.error(msg) f = open(self.configPath, 'w') cPickle.dump(config, f) f.close() # Set up ZMQ try: context = zmq.Context() self.sender = context.socket(zmq.PUSH) self.sender.bind("tcp://*:%s" % inPort) self.sink = context.socket(zmq.PULL) self.sink.bind("tcp://*:%s" % outPort) except zmq.ZMQError: # Try this again in a moment to see # if it's just being held by something pre-existing import time time.sleep(1) logging.error("Blocked socket on startup: Attempting sleep to give it time to clear.") try: context = zmq.Context() self.sender = context.socket(zmq.PUSH) self.sender.bind("tcp://*:%s" % inPort) self.sink = context.socket(zmq.PULL) self.sink.bind("tcp://*:%s" % outPort) except Exception as ex: msg = "Error attempting to open TCP sockets\n" msg += str(ex) logging.error(msg) import traceback print traceback.format_exc() raise ProcessPoolException(msg) # Now actually create the slaves self.createSlaves() return def createSlaves(self): """ _createSlaves_ Create the slaves by using the values from __init__() Moving it into a separate function allows us to restart all of them. """ totalSlaves = self.nSlaves slaveClassName = self.slaveClassName config = self.config namespace = self.namespace inPort = self.inPort outPort = self.outPort slaveArgs = [self.versionString, __file__, self.slaveClassName, inPort, outPort, self.configPath, self.componentDir, self.namespace] count = 0 while totalSlaves > 0: #For each worker you want create a slave process #That process calls this code (WMCore.ProcessPool) and opens #A process pool that loads the designated class slaveProcess = subprocess.Popen(slaveArgs, stdin = subprocess.PIPE, stdout = subprocess.PIPE) self.workers.append(slaveProcess) totalSlaves -= 1 count += 1 return def _subProcessName(self, slaveClassName, sequence): """ subProcessName for heartbeat could change to use process ID as a suffix """ return "%s_%s" % (slaveClassName, sequence + 1) def __del__(self): """ __del__ Kill all the workers processes by sending them an invalid JSON object. This will cause them to shut down. """ self.close() return def close(self): """ _close_ Close shuts down all the active systems by: a) Sending STOP commands for all workers b) Closing the pipes c) Shutting down the workers themselves """ for i in range(self.nSlaves): try: encodedWork = self.jsonHandler.encode('STOP') self.sender.send(encodedWork) except Exception as ex: # Might be already failed. Nothing you can # really do about that. logging.error("Failure killing running process: %s" % str(ex)) pass try: self.sender.close() except: # We can't really do anything if we fail pass try: self.sink.close() except: # We can't do anything if we fail pass # Now close the workers by hand for worker in self.workers: try: worker.join() except Exception as ex: try: worker.terminate() except Exception as ex2: logging.error("Failure to join or terminate process") logging.error(str(ex)) logging.error(str(ex2)) continue self.workers = [] return def enqueue(self, work, list = False): """ __enqeue__ Assign work to the workers processes. The work parameters must be a list where each item in the list can be serialized into JSON. If list is True, the entire list is sent as one piece of work """ if len(self.workers) < 1: # Someone's shut down the system msg = "Attempting to send work after system failure and shutdown!\n" logging.error(msg) raise ProcessPoolException(msg) if not list: for w in work: encodedWork = self.jsonHandler.encode(w) self.sender.send(encodedWork) self.runningWork += 1 else: encodedWork = self.jsonHandler.encode(work) self.sender.send(encodedWork) self.runningWork += 1 return def dequeue(self, totalItems = 1): """ __dequeue__ Retrieve completed work from the slave workers. This method will block until enough work has been completed. """ completedWork = [] if totalItems > self.runningWork: msg = "Asked to dequeue more work then is running!\n" msg += "Failing" logging.error(msg) raise ProcessPoolException(msg) while totalItems > 0: try: output = self.sink.recv() decode = self.jsonHandler.decode(output) if type(decode) == type({}) and decode.get('type', None) == 'ERROR': # Then we had some kind of error msg = decode.get('msg', 'Unknown Error in ProcessPool') logging.error("Received Error Message from ProcessPool Slave") logging.error(msg) self.close() raise ProcessPoolException(msg) completedWork.append(decode) self.runningWork -= 1 totalItems -= 1 except Exception as ex: msg = "Exception while getting slave outputin ProcessPool.\n" msg += str(ex) logging.error(msg) break return completedWork def restart(self): """ _restart_ Delete everything and restart all pools """ self.close() self.createSlaves() return
wmInit = WMInit() setupDB(config, wmInit) # Create JSON handler jsonHandler = JSONRequests() wmFactory = WMFactory(name = "slaveFactory", namespace = namespace) slaveClass = wmFactory.loadObject(classname = slaveClassName, args = config) logging.info("Have slave class") while(True): encodedInput = receiver.recv() try: input = jsonHandler.decode(encodedInput) except Exception as ex: logging.error("Error decoding: %s" % str(ex)) break if input == "STOP": break try: logging.error(input) output = slaveClass(input) except Exception as ex: crashMessage = "Slave process crashed with exception: " + str(ex) crashMessage += "\nStacktrace:\n" stackTrace = traceback.format_tb(sys.exc_info()[2], None)
if __name__ == "__main__": """ __main__ Entry point for the slave process. The slave's classname will be passed in on the command line. The database connection parameters as well as the name of the directory that the log files will be stored in will be passed in through stdin as a JSON object. """ slaveClassName = sys.argv[1] jsonHandler = JSONRequests() encodedConfig = sys.stdin.readline() config = jsonHandler.decode(encodedConfig) encodedSlaveInit = sys.stdin.readline() if encodedSlaveInit != "\n": unicodeSlaveInit = jsonHandler.decode(encodedSlaveInit) slaveInit = {} for key in unicodeSlaveInit.keys(): slaveInit[str(key)] = unicodeSlaveInit[key] else: slaveInit = None wmInit = WMInit() setupLogging(config["componentDir"]) setupDB(config, wmInit) namespace = config.get('namespace', 'WMComponent')