def onGotItem(self, item): """ Process the given item and broadcast outcome Note that item can either be a stop command or a start command not both errors get sent to the logger node and no further processing is done """ d = defer.Deferred() logs = sim.Logger() el = item.firstChildElement() params = sim.getParameters(el, logs) if not params: # not an item for this component d = defer.succeed(None) elif params.cmd == 'stop': d = self.broadcastStop(params) elif params.cmd == 'start': # if they have requested a portfolio run # they should have provided an ID and output location params.run_id = getRunId(item, logs) portfolio = port.getPortfolio(el, logs) if logs.hasSeverity(collab.ERROR_EL): d = self.broadcastLogs(logs, params) else: # broadcast this one log.msg('Portfolio is valid, send on to be processed') d = self.broadcastStart(params, portfolio) return d
def onGotStartSimulation(self, params, item, logger, chunk=100): # get the deferred, add an errback to it and then stick in the cooperator log.msg('sim start', params.run_id) portfolio = port.getPortfolio(item, logger) if not portfolio: yield self.broadcastLogs(logger, params) else: # prep copula try: simulator = self.simulatorFactory['sparse'](portfolio) except Exception as e: yield self._errback(e, logger, params) else: # run a chunk, yielding defaults = defaultdict(int) for count in xrange(0, self.max_runs, chunk): try: simulator.copula(chunk/10, 10, defaults) log.msg('%s done %i' % (params.run_id, count)) except Exception as e: yield self._errback(e, logger, params) else: yield defer.succeed(None) if count > 0 and count%self.broadcast_freq == 0: distributions = sim.Distributions() distributions.combine(collab.DEFAULTS_EL, copy.deepcopy(defaults)) # broadcast out results, yield log.msg( '%s: broadcasting results so far [%s / %s]' % (params.run_id, count, params.number_runs) ) prog = sim.Progress(self.broadcast_freq) d = self.broadcastResults(params, prog, distributions) d.addErrback(self._errback, logger, params) yield d defaults.clear()