Example #1
0
def get_data_transfer(dstore):
    """
    Determine the amount of data transferred from the controller node
    to the workers and back in a classical calculation.

    :param dstore: a :class:`openquake.commonlib.datastore.DataStore` instance
    :returns: (block_info, to_send_forward, to_send_back)
    """
    oqparam = OqParam.from_(dstore.attrs)
    sitecol = dstore['sitecol']
    rlzs_assoc = dstore['rlzs_assoc']
    info = dstore['job_info']
    sources = dstore['composite_source_model'].get_sources()
    num_gsims_by_trt = groupby(rlzs_assoc, operator.itemgetter(0),
                               lambda group: sum(1 for row in group))
    gsims_assoc = rlzs_assoc.gsims_by_trt_id
    to_send_forward = 0
    to_send_back = 0
    block_info = []
    for block in split_in_blocks(sources, oqparam.concurrent_tasks or 1,
                                 operator.attrgetter('weight'),
                                 operator.attrgetter('trt_model_id')):
        num_gsims = num_gsims_by_trt.get(block[0].trt_model_id, 0)
        back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
        to_send_back += back * 8  # 8 bytes per float
        args = (block, sitecol, gsims_assoc, PerformanceMonitor(''))
        to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
        block_info.append((len(block), block.weight))
    return numpy.array(block_info, block_dt), to_send_forward, to_send_back
Example #2
0
def data_transfer(calc):
    """
    Determine the amount of data transferred from the controller node
    to the workers and back in a classical calculation.

    :returns: a triple (num_tasks, to_send_forward, to_send_back)
    """
    oqparam = calc.oqparam
    info = calc.job_info
    calc.monitor.oqparam = oqparam
    sources = calc.composite_source_model.get_sources()
    num_gsims_by_trt = groupby(calc.rlzs_assoc, operator.itemgetter(0),
                               lambda group: sum(1 for row in group))
    gsims_assoc = calc.rlzs_assoc.get_gsims_by_trt_id()
    to_send_forward = 0
    to_send_back = 0
    n_tasks = 0
    for block in split_in_blocks(sources, oqparam.concurrent_tasks,
                                 operator.attrgetter('weight'),
                                 operator.attrgetter('trt_model_id')):
        num_gsims = num_gsims_by_trt[block[0].trt_model_id]
        back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
        to_send_back += back * 8  # 8 bytes per float
        args = (block, calc.sitecol, gsims_assoc, calc.monitor)
        logging.info('Pickling task args #%d', n_tasks)
        to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
        n_tasks += 1
    return n_tasks, to_send_forward, to_send_back
Example #3
0
def get_data_transfer(dstore):
    """
    Determine the amount of data transferred from the controller node
    to the workers and back in a classical calculation.

    :param dstore: a :class:`openquake.commonlib.datastore.DataStore` instance
    :returns: (block_info, to_send_forward, to_send_back)
    """
    oqparam = dstore['oqparam']
    sitecol = dstore['sitecol']
    rlzs_assoc = dstore['rlzs_assoc']
    info = dstore['job_info']
    sources = dstore['composite_source_model'].get_sources()
    num_gsims_by_trt = groupby(rlzs_assoc, operator.itemgetter(0),
                               lambda group: sum(1 for row in group))
    gsims_assoc = rlzs_assoc.gsims_by_trt_id
    to_send_forward = 0
    to_send_back = 0
    block_info = []
    for block in split_in_blocks(sources, oqparam.concurrent_tasks or 1,
                                 operator.attrgetter('weight'),
                                 operator.attrgetter('trt_model_id')):
        num_gsims = num_gsims_by_trt.get(block[0].trt_model_id, 0)
        back = info['n_sites'] * info['n_levels'] * info['n_imts'] * num_gsims
        to_send_back += back * 8  # 8 bytes per float
        args = (block, sitecol, gsims_assoc, parallel.PerformanceMonitor(''))
        to_send_forward += sum(len(p) for p in parallel.pickle_sequence(args))
        block_info.append((len(block), block.weight))
    return numpy.array(block_info, block_dt), to_send_forward, to_send_back
Example #4
0
 def submit(self, *args):
     """
     Submit an oqtask with the given arguments to celery and return
     an AsyncResult. If the variable OQ_NO_DISTRIBUTE is set, the
     task function is run in process and the result is returned.
     """
     check_mem_usage()  # log a warning if too much memory is used
     if no_distribute():
         res = safely_call(self.oqtask.task_func, args)
     else:
         piks = pickle_sequence(args)
         self.sent += sum(len(p) for p in piks)
         res = self.oqtask.delay(*piks)
     self.results.append(res)