def __iter__(self):
self.received = []
if self.num_tasks == 0:
return
for result in self.iresults:
check_mem_usage() # log a warning if too much memory is used
if isinstance(result, BaseException):
# this happens with WorkerLostError with celery
raise result
elif isinstance(result, Result):
val = result.get()
self.received.append(len(result.pik))
else: # this should never happen
raise ValueError(result)
if OQ_DISTRIBUTE == 'processpool':
mem_gb = memory_rss(os.getpid()) + sum(
memory_rss(pid) for pid in Starmap.pids) / GB
else:
mem_gb = numpy.nan
next(self.log_percent)
if not self.name.startswith('_'): # no info for private tasks
self.save_task_info(result.mon, mem_gb)
yield val
if self.received:
tot = sum(self.received)
max_per_task = max(self.received)
self.progress('Received %s of data, maximum per task %s',
humansize(tot), humansize(max_per_task))
def __iter__(self):
if self.iresults == ():
return ()
self.received = []
for result in self.iresults:
check_mem_usage() # log a warning if too much memory is used
if isinstance(result, BaseException):
# this happens with WorkerLostError with celery
raise result
elif isinstance(result, Result):
val = result.get()
self.received.append(len(result.pik))
else: # this should never happen
raise ValueError(result)
if OQ_DISTRIBUTE == 'processpool':
mem_gb = (memory_rss(os.getpid()) +
sum(memory_rss(pid) for pid in Starmap.pids)) / GB
else:
mem_gb = numpy.nan
if not self.name.startswith('_'): # no info for private tasks
self.save_task_info(result.mon, mem_gb)
if result.splice:
yield from val
else:
yield val
if self.received and not self.name.startswith('_'):
tot = sum(self.received)
max_per_output = max(self.received)
msg = 'Received %s from %d outputs, maximum per output %s'
logging.info(msg, humansize(tot), len(self.received),
humansize(max_per_output))
def test_memory(self):
# make sure the memory occupation is low
# (to protect against bad refactoring of the XMLWriter)
pid = os.getpid()
try:
rss = memory_rss(pid)
except psutil.AccessDenied:
raise unittest.SkipTest('Memory info not accessible')
devnull = open(os.devnull, 'wb')
with StreamingXMLWriter(devnull) as writer:
for asset in assetgen(1000):
writer.serialize(asset)
allocated = memory_rss(pid) - rss
self.assertLess(allocated, 256000) # < 250 KB
def test_memory(self):
# make sure the memory occupation is low
# (to protect against bad refactoring of the XMLWriter)
pid = os.getpid()
try:
rss = memory_rss(pid)
except psutil.AccessDenied:
raise unittest.SkipTest('Memory info not accessible')
devnull = open(os.devnull, 'wb')
with StreamingXMLWriter(devnull) as writer:
for asset in assetgen(1000):
writer.serialize(asset)
allocated = memory_rss(pid) - rss
self.assertLess(allocated, 256000) # < 250 KB
def __iter__(self):
if self.iresults == ():
return ()
t0 = time.time()
self.received = []
first_time = True
nbytes = AccumDict()
for result in self.iresults:
msg = check_mem_usage() # log a warning if too much memory is used
if msg and first_time:
logging.warning(msg)
first_time = False # warn only once
if isinstance(result, BaseException):
# this happens with WorkerLostError with celery
raise result
elif isinstance(result, Result):
val = result.get()
self.received.append(len(result.pik))
if hasattr(result, 'nbytes'):
nbytes += result.nbytes
else: # this should never happen
raise ValueError(result)
if OQ_DISTRIBUTE == 'processpool' and sys.platform != 'darwin':
# it normally works on macOS, but not in notebooks calling
# notebooks, which is the case relevant for Marco Pagani
mem_gb = (memory_rss(os.getpid()) +
sum(memory_rss(pid) for pid in Starmap.pids)) / GB
else:
# measure only the memory used by the main process
mem_gb = memory_rss(os.getpid()) / GB
save_task_info(self, result, mem_gb)
if not result.func_args: # not subtask
yield val
if self.received:
tot = sum(self.received)
max_per_output = max(self.received)
logging.info(
'Received %s from %d %s outputs in %d seconds, biggest '
'output=%s', humansize(tot), len(self.received), self.name,
time.time() - t0, humansize(max_per_output))
if nbytes:
logging.info('Received %s',
{k: humansize(v)
for k, v in nbytes.items()})
def __iter__(self):
if self.iresults == ():
return ()
t0 = time.time()
self.received = []
first_time = True
for result in self.iresults:
msg = check_mem_usage() # log a warning if too much memory is used
if msg and first_time:
logging.warn(msg)
first_time = False # warn only once
if isinstance(result, BaseException):
# this happens with WorkerLostError with celery
raise result
elif isinstance(result, Result):
val = result.get()
self.received.append(len(result.pik))
else: # this should never happen
raise ValueError(result)
if OQ_DISTRIBUTE == 'processpool' and sys.platform != 'darwin':
# it normally works on macOS, but not in notebooks calling
# notebooks, which is the case relevant for Marco Pagani
mem_gb = (memory_rss(os.getpid()) +
sum(memory_rss(pid) for pid in Starmap.pids)) / GB
else:
mem_gb = numpy.nan
self.save_task_info(result.mon, mem_gb)
if result.splice:
yield from val
else:
yield val
if self.received:
tot = sum(self.received)
max_per_output = max(self.received)
logging.info(
'Received %s from %d outputs in %d seconds, biggest '
'output=%s', humansize(tot), len(self.received),
time.time() - t0, humansize(max_per_output))