def test_no_distribute_set_false(self):
with patch.dict('os.environ'):
os.environ['OQ_DISTRIBUTE'] = 'celery'
self.assertFalse(no_distribute())
os.environ['OQ_DISTRIBUTE'] = 'futures'
self.assertFalse(no_distribute())
os.environ['OQ_DISTRIBUTE'] = 'blarg'
self.assertFalse(no_distribute())
def test_no_distribute_set_false(self):
with patch.dict('os.environ'):
os.environ['OQ_DISTRIBUTE'] = 'celery'
self.assertFalse(no_distribute())
os.environ['OQ_DISTRIBUTE'] = 'futures'
self.assertFalse(no_distribute())
os.environ['OQ_DISTRIBUTE'] = 'blarg'
self.assertFalse(no_distribute())
def submit(self, *args):
"""
Submit an oqtask with the given arguments to celery and return
an AsyncResult. If the variable OQ_NO_DISTRIBUTE is set, the
task function is run in process and the result is returned.
"""
check_mem_usage() # log a warning if too much memory is used
if no_distribute():
res = safely_call(self.oqtask.task_func, args)
else:
piks = pickle_sequence(args)
self.sent += sum(len(p) for p in piks)
res = self.oqtask.delay(*piks)
self.results.append(res)
def get_composite_source_model(oqparam,
sitecol=None,
SourceProcessor=source.SourceFilterSplitter,
monitor=DummyMonitor(),
no_distribute=parallel.no_distribute()):
"""
Build the source models by splitting the sources. If prefiltering is
enabled, also reduce the GSIM logic trees in the underlying source models.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param SourceProcessor:
the class used to process the sources
:param monitor:
a monitor instance
:param no_distribute:
used to disable parallel splitting of the sources
:returns:
an iterator over :class:`openquake.commonlib.source.SourceModel`
"""
processor = SourceProcessor(sitecol, oqparam.maximum_distance,
oqparam.area_source_discretization)
source_model_lt = get_source_model_lt(oqparam)
smodels = []
trt_id = 0
for source_model in get_source_models(oqparam,
source_model_lt,
processor.sitecol,
in_memory=hasattr(
processor, 'process')):
for trt_model in source_model.trt_models:
trt_model.id = trt_id
trt_id += 1
smodels.append(source_model)
csm = source.CompositeSourceModel(source_model_lt, smodels)
if sitecol is not None and hasattr(processor, 'process'):
seqtime, partime = processor.process(csm, no_distribute)
monitor.write(['fast sources filtering/splitting', str(seqtime), '0'])
monitor.write(['slow sources filtering/splitting', str(partime), '0'])
if not csm.get_sources():
raise RuntimeError('All sources were filtered away')
csm.count_ruptures()
return csm
def get_composite_source_model(
oqparam, sitecol=None, SourceProcessor=source.SourceFilterSplitter,
monitor=DummyMonitor(), no_distribute=parallel.no_distribute()):
"""
Build the source models by splitting the sources. If prefiltering is
enabled, also reduce the GSIM logic trees in the underlying source models.
:param oqparam:
an :class:`openquake.commonlib.oqvalidation.OqParam` instance
:param sitecol:
a :class:`openquake.hazardlib.site.SiteCollection` instance
:param SourceProcessor:
the class used to process the sources
:param monitor:
a monitor instance
:param no_distribute:
used to disable parallel splitting of the sources
:returns:
an iterator over :class:`openquake.commonlib.source.SourceModel`
"""
processor = SourceProcessor(sitecol, oqparam.maximum_distance,
oqparam.area_source_discretization)
source_model_lt = get_source_model_lt(oqparam)
smodels = []
trt_id = 0
for source_model in get_source_models(
oqparam, source_model_lt, processor.sitecol,
in_memory=hasattr(processor, 'process')):
for trt_model in source_model.trt_models:
trt_model.id = trt_id
trt_id += 1
smodels.append(source_model)
csm = source.CompositeSourceModel(source_model_lt, smodels)
if sitecol is not None and hasattr(processor, 'process'):
seqtime, partime = processor.process(csm, no_distribute)
monitor.write(['fast sources filtering/splitting', str(seqtime), '0'])
monitor.write(['slow sources filtering/splitting', str(partime), '0'])
if not csm.get_sources():
raise RuntimeError('All sources were filtered away')
csm.count_ruptures()
return csm
def test_no_distribute_set_true(self):
with patch.dict('os.environ'):
os.environ['OQ_DISTRIBUTE'] = 'no'
self.assertTrue(no_distribute())
def test_no_distribute_not_set(self):
with patch.dict('os.environ'):
if 'OQ_DISTRIBUTE' in os.environ:
os.environ.pop('OQ_DISTRIBUTE')
self.assertFalse(no_distribute())
def test_no_distribute_set_true(self):
with patch.dict('os.environ'):
os.environ['OQ_DISTRIBUTE'] = 'no'
self.assertTrue(no_distribute())
def test_no_distribute_not_set(self):
with patch.dict('os.environ'):
if 'OQ_DISTRIBUTE' in os.environ:
os.environ.pop('OQ_DISTRIBUTE')
self.assertFalse(no_distribute())
