def test_no_distribute_set_false(self): with patch.dict('os.environ'): os.environ['OQ_DISTRIBUTE'] = 'celery' self.assertFalse(no_distribute()) os.environ['OQ_DISTRIBUTE'] = 'futures' self.assertFalse(no_distribute()) os.environ['OQ_DISTRIBUTE'] = 'blarg' self.assertFalse(no_distribute())
def submit(self, *args): """ Submit an oqtask with the given arguments to celery and return an AsyncResult. If the variable OQ_NO_DISTRIBUTE is set, the task function is run in process and the result is returned. """ check_mem_usage() # log a warning if too much memory is used if no_distribute(): res = safely_call(self.oqtask.task_func, args) else: piks = pickle_sequence(args) self.sent += sum(len(p) for p in piks) res = self.oqtask.delay(*piks) self.results.append(res)
def get_composite_source_model(oqparam, sitecol=None, SourceProcessor=source.SourceFilterSplitter, monitor=DummyMonitor(), no_distribute=parallel.no_distribute()): """ Build the source models by splitting the sources. If prefiltering is enabled, also reduce the GSIM logic trees in the underlying source models. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param SourceProcessor: the class used to process the sources :param monitor: a monitor instance :param no_distribute: used to disable parallel splitting of the sources :returns: an iterator over :class:`openquake.commonlib.source.SourceModel` """ processor = SourceProcessor(sitecol, oqparam.maximum_distance, oqparam.area_source_discretization) source_model_lt = get_source_model_lt(oqparam) smodels = [] trt_id = 0 for source_model in get_source_models(oqparam, source_model_lt, processor.sitecol, in_memory=hasattr( processor, 'process')): for trt_model in source_model.trt_models: trt_model.id = trt_id trt_id += 1 smodels.append(source_model) csm = source.CompositeSourceModel(source_model_lt, smodels) if sitecol is not None and hasattr(processor, 'process'): seqtime, partime = processor.process(csm, no_distribute) monitor.write(['fast sources filtering/splitting', str(seqtime), '0']) monitor.write(['slow sources filtering/splitting', str(partime), '0']) if not csm.get_sources(): raise RuntimeError('All sources were filtered away') csm.count_ruptures() return csm
def get_composite_source_model( oqparam, sitecol=None, SourceProcessor=source.SourceFilterSplitter, monitor=DummyMonitor(), no_distribute=parallel.no_distribute()): """ Build the source models by splitting the sources. If prefiltering is enabled, also reduce the GSIM logic trees in the underlying source models. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param sitecol: a :class:`openquake.hazardlib.site.SiteCollection` instance :param SourceProcessor: the class used to process the sources :param monitor: a monitor instance :param no_distribute: used to disable parallel splitting of the sources :returns: an iterator over :class:`openquake.commonlib.source.SourceModel` """ processor = SourceProcessor(sitecol, oqparam.maximum_distance, oqparam.area_source_discretization) source_model_lt = get_source_model_lt(oqparam) smodels = [] trt_id = 0 for source_model in get_source_models( oqparam, source_model_lt, processor.sitecol, in_memory=hasattr(processor, 'process')): for trt_model in source_model.trt_models: trt_model.id = trt_id trt_id += 1 smodels.append(source_model) csm = source.CompositeSourceModel(source_model_lt, smodels) if sitecol is not None and hasattr(processor, 'process'): seqtime, partime = processor.process(csm, no_distribute) monitor.write(['fast sources filtering/splitting', str(seqtime), '0']) monitor.write(['slow sources filtering/splitting', str(partime), '0']) if not csm.get_sources(): raise RuntimeError('All sources were filtered away') csm.count_ruptures() return csm
def test_no_distribute_set_true(self): with patch.dict('os.environ'): os.environ['OQ_DISTRIBUTE'] = 'no' self.assertTrue(no_distribute())
def test_no_distribute_not_set(self): with patch.dict('os.environ'): if 'OQ_DISTRIBUTE' in os.environ: os.environ.pop('OQ_DISTRIBUTE') self.assertFalse(no_distribute())