def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
"""
Run the calculation and return the exported outputs.
"""
global logversion
with self._monitor:
self.close = close
self.set_log_format()
if logversion: # make sure this is logged only once
logging.info('Running %s', self.oqparam.inputs['job_ini'])
logging.info('Using engine version %s', engine_version)
logversion = False
if concurrent_tasks is None: # use the job.ini parameter
ct = self.oqparam.concurrent_tasks
else: # used the parameter passed in the command-line
ct = concurrent_tasks
if ct == 0: # disable distribution temporarily
oq_distribute = os.environ.get('OQ_DISTRIBUTE')
os.environ['OQ_DISTRIBUTE'] = 'no'
if ct != self.oqparam.concurrent_tasks:
# save the used concurrent_tasks
self.oqparam.concurrent_tasks = ct
self.save_params(**kw)
Starmap.init()
try:
if pre_execute:
self.pre_execute()
self.result = self.execute()
if self.result is not None:
self.post_execute(self.result)
self.before_export()
self.export(kw.get('exports', ''))
except Exception:
if kw.get('pdb'): # post-mortem debug
tb = sys.exc_info()[2]
traceback.print_tb(tb)
pdb.post_mortem(tb)
else:
logging.critical('', exc_info=True)
raise
finally:
# cleanup globals
if ct == 0: # restore OQ_DISTRIBUTE
if oq_distribute is None: # was not set
del os.environ['OQ_DISTRIBUTE']
else:
os.environ['OQ_DISTRIBUTE'] = oq_distribute
readinput.pmap = None
readinput.exposure = None
Starmap.shutdown()
self._monitor.flush()
return getattr(self, 'exported', {})
def pfilter(self, sources, monitor):
"""
Filter the sources in parallel by using Starmap.apply
:param sources: a sequence of sources
:param monitor: a Monitor instance
:returns: a dictionary src_group_id -> sources
"""
sources_by_grp = Starmap.apply(prefilter, (sources, self, monitor),
distribute=self.distribute,
name=self.__class__.__name__).reduce()
Starmap.shutdown() # close the processpool
Starmap.init() # reopen it when necessary
# avoid task ordering issues
for sources in sources_by_grp.values():
sources.sort(key=operator.attrgetter('source_id'))
return sources_by_grp
