def save_from(self, source: typing.Generator, rechunk=True, executor=None):
"""Iterate over source and save the results under key
along with metadata
"""
pending = []
exhausted = False
chunk_i = 0
try:
while not exhausted:
chunk = None
try:
if rechunk and self.allow_rechunk:
while (chunk is None or
chunk.data.nbytes < chunk.target_size_mb * 1e6):
chunk = strax.Chunk.concatenate(
[chunk, next(source)])
else:
chunk = next(source)
except StopIteration:
exhausted = True
if chunk is None:
break
new_f = self.save(chunk=chunk,
chunk_i=chunk_i,
executor=executor)
pending = [f for f in pending if not f.done()]
if new_f is not None:
pending += [new_f]
chunk_i += 1
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close(wait_for=pending)
pass
except Exception as e:
# log exception for the final check
self.got_exception = e
# Throw the exception back into the mailbox
# (hoping that it is still listening...)
source.throw(e)
raise e
finally:
if not self.closed:
self.close(wait_for=pending)
def save_from(self, source: typing.Generator, rechunk=True, executor=None):
"""Iterate over source and save the results under key
along with metadata
"""
pending = []
exhausted = False
chunk_i = 0
run_id = self.md['run_id']
_is_super_run = run_id.startswith('_')
try:
while not exhausted:
chunk = None
try:
if rechunk and self.allow_rechunk:
while (chunk is None or
chunk.data.nbytes < chunk.target_size_mb*1e6):
next_chunk = next(source)
if _is_super_run:
# If we are creating a superrun, we load data from subruns
# and the loaded subrun chunk becomes a superun chunk:
next_chunk = strax.transform_chunk_to_superrun_chunk(run_id,
next_chunk)
chunk = strax.Chunk.concatenate([chunk, next_chunk])
else:
chunk = next(source)
if _is_super_run:
# If we are creating a superrun, we load data from subruns
# and the loaded subrun chunk becomes a superun chunk:
chunk = strax.transform_chunk_to_superrun_chunk(run_id, chunk)
except StopIteration:
exhausted = True
if chunk is None:
break
new_f = self.save(chunk=chunk,
chunk_i=chunk_i, executor=executor)
pending = [f for f in pending if not f.done()]
if new_f is not None:
pending += [new_f]
chunk_i += 1
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close(wait_for=pending)
except Exception as e:
# log exception for the final check
self.got_exception = e
# Throw the exception back into the mailbox
# (hoping that it is still listening...)
source.throw(e)
raise e
finally:
if not self.closed:
self.close(wait_for=pending)