def get_data_transfer(dstore):
"""
Determine the amount of data transferred from the controller node
to the workers and back in a classical calculation.
:param dstore: a :class:`openquake.commonlib.datastore.DataStore` instance
:returns: (block_info, to_send_forward, to_send_back)
"""
oqparam = OqParam.from_(dstore.attrs)
sitecol = dstore['sitecol']
rlzs_assoc = dstore['rlzs_assoc']
info = dstore['job_info']
sources = dstore['composite_source_model'].get_sources()
num_gsims_by_trt = groupby(rlzs_assoc, operator.itemgetter(0),
lambda group: sum(1 for row in group))
gsims_assoc = rlzs_assoc.gsims_by_trt_id
to_send_forward = 0
to_send_back = 0
block_info = []
for block in split_in_blocks(sources, oqparam.concurrent_tasks or 1,
operator.attrgetter('weight'),
operator.attrgetter('trt_model_id')):
num_gsims = num_gsims_by_trt.get(block[0].trt_model_id, 0)
back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
to_send_back += back * 8 # 8 bytes per float
args = (block, sitecol, gsims_assoc, PerformanceMonitor(''))
to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
block_info.append((len(block), block.weight))
return numpy.array(block_info, block_dt), to_send_forward, to_send_back
def data_transfer(calc):
"""
Determine the amount of data transferred from the controller node
to the workers and back in a classical calculation.
:returns: a triple (num_tasks, to_send_forward, to_send_back)
"""
oqparam = calc.oqparam
info = calc.job_info
calc.monitor.oqparam = oqparam
sources = calc.composite_source_model.get_sources()
num_gsims_by_trt = groupby(calc.rlzs_assoc, operator.itemgetter(0),
lambda group: sum(1 for row in group))
gsims_assoc = calc.rlzs_assoc.get_gsims_by_trt_id()
to_send_forward = 0
to_send_back = 0
n_tasks = 0
for block in split_in_blocks(sources, oqparam.concurrent_tasks,
operator.attrgetter('weight'),
operator.attrgetter('trt_model_id')):
num_gsims = num_gsims_by_trt[block[0].trt_model_id]
back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
to_send_back += back * 8 # 8 bytes per float
args = (block, calc.sitecol, gsims_assoc, calc.monitor)
logging.info('Pickling task args #%d', n_tasks)
to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
n_tasks += 1
return n_tasks, to_send_forward, to_send_back
def get_data_transfer(dstore):
"""
Determine the amount of data transferred from the controller node
to the workers and back in a classical calculation.
:param dstore: a :class:`openquake.commonlib.datastore.DataStore` instance
:returns: (block_info, to_send_forward, to_send_back)
"""
oqparam = dstore['oqparam']
sitecol = dstore['sitecol']
rlzs_assoc = dstore['rlzs_assoc']
info = dstore['job_info']
sources = dstore['composite_source_model'].get_sources()
num_gsims_by_trt = groupby(rlzs_assoc, operator.itemgetter(0),
lambda group: sum(1 for row in group))
gsims_assoc = rlzs_assoc.gsims_by_trt_id
to_send_forward = 0
to_send_back = 0
block_info = []
for block in split_in_blocks(sources, oqparam.concurrent_tasks or 1,
operator.attrgetter('weight'),
operator.attrgetter('trt_model_id')):
num_gsims = num_gsims_by_trt.get(block[0].trt_model_id, 0)
back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
to_send_back += back * 8 # 8 bytes per float
args = (block, sitecol, gsims_assoc, parallel.PerformanceMonitor(''))
to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
block_info.append((len(block), block.weight))
return numpy.array(block_info, block_dt), to_send_forward, to_send_back
def submit(self, *args):
"""
Submit an oqtask with the given arguments to celery and return
an AsyncResult. If the variable OQ_NO_DISTRIBUTE is set, the
task function is run in process and the result is returned.
"""
check_mem_usage() # log a warning if too much memory is used
if no_distribute():
res = safely_call(self.oqtask.task_func, args)
else:
piks = pickle_sequence(args)
self.sent += sum(len(p) for p in piks)
res = self.oqtask.delay(*piks)
self.results.append(res)