def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw): """ Run the calculation and return the exported outputs. """ global logversion with self._monitor: self.close = close self.set_log_format() if logversion: # make sure this is logged only once logging.info('Running %s', self.oqparam.inputs['job_ini']) logging.info('Using engine version %s', engine_version) logversion = False if concurrent_tasks is None: # use the job.ini parameter ct = self.oqparam.concurrent_tasks else: # used the parameter passed in the command-line ct = concurrent_tasks if ct == 0: # disable distribution temporarily oq_distribute = os.environ.get('OQ_DISTRIBUTE') os.environ['OQ_DISTRIBUTE'] = 'no' if ct != self.oqparam.concurrent_tasks: # save the used concurrent_tasks self.oqparam.concurrent_tasks = ct self.save_params(**kw) Starmap.init() try: if pre_execute: self.pre_execute() self.result = self.execute() if self.result is not None: self.post_execute(self.result) self.before_export() self.export(kw.get('exports', '')) except Exception: if kw.get('pdb'): # post-mortem debug tb = sys.exc_info()[2] traceback.print_tb(tb) pdb.post_mortem(tb) else: logging.critical('', exc_info=True) raise finally: # cleanup globals if ct == 0: # restore OQ_DISTRIBUTE if oq_distribute is None: # was not set del os.environ['OQ_DISTRIBUTE'] else: os.environ['OQ_DISTRIBUTE'] = oq_distribute readinput.pmap = None readinput.exposure = None Starmap.shutdown() self._monitor.flush() return getattr(self, 'exported', {})
def pfilter(self, sources, monitor): """ Filter the sources in parallel by using Starmap.apply :param sources: a sequence of sources :param monitor: a Monitor instance :returns: a dictionary src_group_id -> sources """ sources_by_grp = Starmap.apply(prefilter, (sources, self, monitor), distribute=self.distribute, name=self.__class__.__name__).reduce() Starmap.shutdown() # close the processpool Starmap.init() # reopen it when necessary # avoid task ordering issues for sources in sources_by_grp.values(): sources.sort(key=operator.attrgetter('source_id')) return sources_by_grp