def _graduateCmd(self, cmd, cluster, fail, custom): # Cluster callback needs to passthrough this object, # so that we know when a cluster is finished, otherwise # we can't mark ourselves as needing work. # Alternatively, "need for work" can be defined by polling # and checking some mix of processing load and queue length. # custom is of the form: #{'filename.nc': ["('http://host:8082/pathname/munged.nc', 1234)"]} # FIXME: don't forget to do the unregistering. if custom: def unbundle(x): props = x[1][0] # Want to do eval(x[1][0]), but it's unsafe. props = props[1:-1].split(', ') # Drop the parens and split. return (x[0], props[0][1:-1], int(props[1])) cmd.actualOutputs = [unbundle(x) for x in custom.items()] log.debug("Remote cmd produced %s" %(str(cmd.actualOutputs))) self.actual.update([(x[0],x[1]) for x in cmd.actualOutputs]) else: cmd.actualOutputs = [] log.warning("Remote cmd produced no outputs") # Do cluster bookkeeping cluster.exec_finishCount += 1 if cluster.exec_finishCount == cluster.exec_criticalCount: # Request deferred discard. cluster.exec_finishFunc() self.runningClusters.discard(cluster) #discard supresses errors. files = chain(*imap(lambda c: c.outputs, cluster.deferred)) self.rpc.discardFiles([x for x in files]) self.finishedClusters.add(cluster) pass
def dropWorker(self, executor): if executor in self.executor: log.info("Removing worker " + executor.url ) self.executor.remove(executor) return True else: log.warning("Tried, but couldn't remove " + executor.url) return True
def writePid(self, filename=None): if not filename: filename = self.servicePidFile try: open(filename, "w").write(str(os.getpid())+"\n") except: log.warning("Couldn't write pid to %s" % filename) pass
def pollState(self, token): """ this can be merged soon""" if token not in self.jobs: time.sleep(0.2) # possible race if token not in self.jobs: log.warning("token not ready after waiting.") if self.config.serviceMode == "master": return SwampTaskState.newState(token, "missing").packed() else: return None if self.config.serviceMode != "master": log.error("pollState not implemented here yet") stateObject = self._taskStateObject(self.jobs[token]) stateObject.token = token if not stateObject: log.error("SOAP interface found weird object in self.jobs:" + "token(%d) has %s" %(token, str(self.jobs[token])) ) return SwampTaskState.newState(token, "system error").packed() else: return stateObject.packed()
def pollStats(self, token): if token not in self.jobs: time.sleep(0.2) # possible race if token not in self.jobs: log.warning("token not ready after waiting.")
def update(self, overrides): if overrides: log.warning("Unimplemented configuration overriding code.") pass