Ejemplo n.º 1
0
    def save_from(self, source: typing.Iterable, rechunk=True, executor=None):
        """Iterate over source and save the results under key
        along with metadata
        """
        if rechunk and self.allow_rechunk:
            source = strax.fixed_size_chunks(source)

        pending = []
        try:
            for chunk_i, s in enumerate(source):
                new_f = self.save(data=s, chunk_i=chunk_i, executor=executor)
                if new_f is not None:
                    pending = [f for f in pending + [new_f] if not f.done()]

        except strax.MailboxKilled:
            # Write exception (with close), but exit gracefully.
            # One traceback on screen is enough
            self.close(wait_for=pending)
            pass

        except Exception as e:
            # log exception for the final check
            self.got_exception = e
            # Throw the exception back into the mailbox
            # (hoping that it is still listening...)
            source.throw(e)
            raise e

        finally:
            if not self.closed:
                self.close(wait_for=pending)
Ejemplo n.º 2
0
    def save_from(self, source: typing.Iterable, rechunk=True):
        """Iterate over source and save the results under key
        along with metadata
        """
        if rechunk and self.prefer_rechunk:
            source = strax.fixed_size_chunks(source)

        try:
            for chunk_i, s in enumerate(source):
                self.save(data=s, chunk_i=chunk_i)
        except strax.MailboxKilled:
            # Write exception (with close), but exit gracefully.
            # One traceback on screen is enough
            self.close()
            pass
        finally:
            if not self.closed:
                self.close()
Ejemplo n.º 3
0
    def save_from(self, source: typing.Iterable, rechunk=True, executor=None):
        """Iterate over source and save the results under key
        along with metadata
        """
        if rechunk and self.prefer_rechunk:
            source = strax.fixed_size_chunks(source)

        pending = []
        try:
            for chunk_i, s in enumerate(source):
                new_f = self.save(data=s, chunk_i=chunk_i, executor=executor)
                if new_f is not None:
                    pending = [f for f in pending + [new_f] if not f.done()]

        except strax.MailboxKilled:
            # Write exception (with close), but exit gracefully.
            # One traceback on screen is enough
            self.close(wait_for=pending)
            pass
        finally:
            if not self.closed:
                self.close(wait_for=pending)
Ejemplo n.º 4
0
def test_fixed_size_chunks_oversized(source):
    # test chunk size > array
    result = list(strax.fixed_size_chunks(source, int(1e9)))
    _check_mangling(result)
Ejemplo n.º 5
0
def test_fixed_size_chunks(source):
    # test chunk size < array
    result = list(strax.fixed_size_chunks(source, 42 * 8))
    assert np.all(np.array([len(x) for x in result[:-1]]) == 42)
    assert len(result[-1]) == 1000 % 42
    _check_mangling(result)