def provide(self, request):
report_next_timeout = 2
num_rejected = 0
timing = Timing(self)
timing.start()
have_good_batch = False
while not have_good_batch:
batch = self.upstream_provider.request_batch(request)
if batch.arrays[self.ensure_nonempty].data.size != 0:
have_good_batch = True
logger.debug("Accepted batch with shape: %s",
batch.arrays[self.ensure_nonempty].data.shape)
else:
num_rejected += 1
if timing.elapsed() > report_next_timeout:
logger.info(
"rejected %s batches, been waiting for a good one "
"since %s", num_rejected, report_next_timeout)
report_next_timeout *= 2
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
report_next_timeout = 10
num_rejected = 0
timing = Timing(self)
timing.start()
assert self.mask in request, (
"Reject can only be used if a GT mask is requested")
have_good_batch = False
while not have_good_batch:
batch = self.upstream_provider.request_batch(request)
mask_ratio = batch.arrays[self.mask].data.mean()
have_good_batch = mask_ratio > self.min_masked
if not have_good_batch and self.reject_probability < 1.:
have_good_batch = random.random() > self.reject_probability
if not have_good_batch:
logger.debug("reject batch with mask ratio %f at %s",
mask_ratio, batch.arrays[self.mask].spec.roi)
num_rejected += 1
if timing.elapsed() > report_next_timeout:
logger.warning(
"rejected %d batches, been waiting for a good one "
"since %ds", num_rejected, report_next_timeout)
report_next_timeout *= 2
else:
logger.debug("accepted batch with mask ratio %f at %s",
mask_ratio, batch.arrays[self.mask].spec.roi)
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
report_next_timeout = 10
num_rejected = 0
timing = Timing(self)
timing.start()
assert self.mask_volume_type in request.volumes, "Reject can only be used if a GT mask is requested"
have_good_batch = False
while not have_good_batch:
batch = self.upstream_provider.request_batch(request)
mask_ratio = batch.volumes[self.mask_volume_type].data.mean()
have_good_batch = mask_ratio >= self.min_masked
if not have_good_batch:
logger.debug("reject batch with mask ratio %f at " %
mask_ratio +
str(batch.volumes[self.mask_volume_type].roi))
num_rejected += 1
if timing.elapsed() > report_next_timeout:
logger.warning(
"rejected %d batches, been waiting for a good one since %ds"
% (num_rejected, report_next_timeout))
report_next_timeout *= 2
logger.debug("good batch with mask ratio %f found at " % mask_ratio +
str(batch.volumes[self.mask_volume_type].roi))
timing.stop()
batch.profiling_stats.add(timing)
return batch
def provide(self, request):
random.seed(request.random_seed)
report_next_timeout = 10
num_rejected = 0
timing = Timing(self)
timing.start()
if self.mask:
assert self.mask in request, (
"Reject can only be used if %s is provided" % self.mask)
if self.ensure_nonempty:
assert self.ensure_nonempty in request, (
"Reject can only be used if %s is provided" %
self.ensure_nonempty)
have_good_batch = False
while not have_good_batch:
batch = self.upstream_provider.request_batch(request)
if self.mask:
mask_ratio = batch.arrays[self.mask].data.mean()
else:
mask_ratio = None
if self.ensure_nonempty:
num_points = len(list(
batch.points[self.ensure_nonempty].nodes))
else:
num_points = None
have_min_mask = mask_ratio is None or mask_ratio > self.min_masked
have_points = num_points is None or num_points > 0
have_good_batch = have_min_mask and have_points
if not have_good_batch and self.reject_probability < 1.:
have_good_batch = random.random() > self.reject_probability
if not have_good_batch:
if self.mask:
logger.debug("reject batch with mask ratio %f at %s",
mask_ratio, batch.arrays[self.mask].spec.roi)
if self.ensure_nonempty:
logger.debug("reject batch with empty points in %s",
batch.points[self.ensure_nonempty].spec.roi)
num_rejected += 1
if timing.elapsed() > report_next_timeout:
logger.warning(
"rejected %d batches, been waiting for a good one "
"since %ds", num_rejected, report_next_timeout)
report_next_timeout *= 2
else:
if self.mask:
logger.debug("accepted batch with mask ratio %f at %s",
mask_ratio, batch.arrays[self.mask].spec.roi)
if self.ensure_nonempty:
logger.debug("accepted batch with nonempty points in %s",
self.ensure_nonempty)
timing.stop()
batch.profiling_stats.add(timing)
return batch
class PrintProfilingStats(BatchFilter):
'''Print profiling information about nodes upstream of this node in the DAG.
The output also includes a ``TOTAL`` section, which shows the wall-time
spent in the upstream and downstream passes. For the downstream pass, this
information is not available in the first iteration, since the request-batch
cycle is not completed, yet.
Args:
every (``int``):
Collect statistics about that many batch requests and show min,
max, mean, and median runtimes.
'''
def __init__(self, every=1):
self.every = every
self.n = 0
self.accumulated_stats = ProfilingStats()
self.__upstream_timing = Timing(self)
self.__upstream_timing_summary = TimingSummary()
self.__downstream_timing = Timing(self)
self.__downstream_timing_summary = TimingSummary()
def prepare(self, request):
self.__downstream_timing.stop()
# skip the first one, where we don't know how much time we spent
# downstream
if self.__downstream_timing.elapsed() > 0:
self.__downstream_timing_summary.add(self.__downstream_timing)
self.__downstream_timing = Timing(self)
self.__upstream_timing.start()
def process(self, batch, request):
self.__upstream_timing.stop()
self.__upstream_timing_summary.add(self.__upstream_timing)
self.__upstream_timing = Timing(self)
self.__downstream_timing.start()
self.n += 1
print_stats = self.n % self.every == 0
self.accumulated_stats.merge_with(batch.profiling_stats)
if not print_stats:
return
span_start, span_end = self.accumulated_stats.span()
stats = "\n"
stats += "Profiling Stats\n"
stats += "===============\n"
stats += "\n"
stats += "NODE".ljust(20)
stats += "METHOD".ljust(10)
stats += "COUNTS".ljust(10)
stats += "MIN".ljust(10)
stats += "MAX".ljust(10)
stats += "MEAN".ljust(10)
stats += "MEDIAN".ljust(10)
stats += "\n"
summaries = list(self.accumulated_stats.get_timing_summaries().items())
summaries.sort()
for (node_name, method_name), summary in summaries:
if summary.counts() > 0:
stats += node_name[:19].ljust(20)
stats += method_name[:19].ljust(
10) if method_name is not None else ' ' * 10
stats += ("%d" % summary.counts())[:9].ljust(10)
stats += ("%.2f" % summary.min())[:9].ljust(10)
stats += ("%.2f" % summary.max())[:9].ljust(10)
stats += ("%.2f" % summary.mean())[:9].ljust(10)
stats += ("%.2f" % summary.median())[:9].ljust(10)
stats += "\n"
stats += "\n"
stats += "TOTAL"
stats += "\n"
for phase, summary in zip(
['upstream', 'downstream'],
[self.__upstream_timing_summary, self.__downstream_timing_summary
]):
if summary.counts() > 0:
stats += phase[:19].ljust(30)
stats += ("%d" % summary.counts())[:9].ljust(10)
stats += ("%.2f" % summary.min())[:9].ljust(10)
stats += ("%.2f" % summary.max())[:9].ljust(10)
stats += ("%.2f" % summary.mean())[:9].ljust(10)
stats += ("%.2f" % summary.median())[:9].ljust(10)
stats += "\n"
stats += "\n"
logger.info(stats)
# reset summaries
self.accumulated_stats = ProfilingStats()
self.__upstream_timing_summary = TimingSummary()
self.__downstream_timing_summary = TimingSummary()