def save_from(self, source: typing.Iterable, rechunk=True, executor=None):
"""Iterate over source and save the results under key
along with metadata
"""
if rechunk and self.allow_rechunk:
source = strax.fixed_size_chunks(source)
pending = []
try:
for chunk_i, s in enumerate(source):
new_f = self.save(data=s, chunk_i=chunk_i, executor=executor)
if new_f is not None:
pending = [f for f in pending + [new_f] if not f.done()]
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close(wait_for=pending)
pass
except Exception as e:
# log exception for the final check
self.got_exception = e
# Throw the exception back into the mailbox
# (hoping that it is still listening...)
source.throw(e)
raise e
finally:
if not self.closed:
self.close(wait_for=pending)
def save_from(self, source: typing.Iterable, rechunk=True):
"""Iterate over source and save the results under key
along with metadata
"""
if rechunk and self.prefer_rechunk:
source = strax.fixed_size_chunks(source)
try:
for chunk_i, s in enumerate(source):
self.save(data=s, chunk_i=chunk_i)
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close()
pass
finally:
if not self.closed:
self.close()
def save_from(self, source: typing.Iterable, rechunk=True, executor=None):
"""Iterate over source and save the results under key
along with metadata
"""
if rechunk and self.prefer_rechunk:
source = strax.fixed_size_chunks(source)
pending = []
try:
for chunk_i, s in enumerate(source):
new_f = self.save(data=s, chunk_i=chunk_i, executor=executor)
if new_f is not None:
pending = [f for f in pending + [new_f] if not f.done()]
except strax.MailboxKilled:
# Write exception (with close), but exit gracefully.
# One traceback on screen is enough
self.close(wait_for=pending)
pass
finally:
if not self.closed:
self.close(wait_for=pending)
def test_fixed_size_chunks_oversized(source):
# test chunk size > array
result = list(strax.fixed_size_chunks(source, int(1e9)))
_check_mangling(result)
def test_fixed_size_chunks(source):
# test chunk size < array
result = list(strax.fixed_size_chunks(source, 42 * 8))
assert np.all(np.array([len(x) for x in result[:-1]]) == 42)
assert len(result[-1]) == 1000 % 42
_check_mangling(result)