Ejemplo n.º 1
0
def certain_kind_tap(data_items):
    """
    As the stream of data items go by, get different
    kinds of information from them, in this case,
    the things that are fruit and metal, collecting
    each kind with a different spigot.

    stream_tap doesn't consume the data_items iterator
    by itself, it's a generator and must be consumed
    by something else. In this case, it's consuming
    the items by casting the iterator to a tuple,
    but doing it in batches.

    Since each batch is not referenced by anything
    the memory can be freed by the garbage collector,
    so no matter the size of the data_items, only a little
    memory is needed. The only things retained
    are the results, which should just be a subset
    of the items and in this case, the getter functions
    only return a portion of each item it matches.


    :param data_items: A sequence of unicode strings
    """
    fruit_spigot = Bucket(get_fruit)
    metal_spigot = Bucket(get_metal)

    items = stream_tap((fruit_spigot, metal_spigot), data_items)

    for batch in i_batch(100, items):
        tuple(batch)

    return fruit_spigot.contents(), metal_spigot.contents()
Ejemplo n.º 2
0
    def test_stream_tap_one_bucket(self):
        """
        Test stream_tap with one bucket.
        """
        odd_bucket = Bucket(get_odd)
        items = stream_tap((odd_bucket,), self.stream)
        out_flow = tuple(items)
        self.assertEqual((1, 2, 3, 4), out_flow)

        self.assertEqual((1, 3), tuple(odd_bucket.contents()))
Ejemplo n.º 3
0
    def test_stream_tap_two_bucket(self):
        """
        Test stream_tap with two buckets.
        """
        odd_bucket = Bucket(get_odd)
        company_bucket = Bucket(get_company_crowd)
        items = stream_tap((odd_bucket, company_bucket), self.stream)
        out_flow = tuple(items)
        self.assertEqual((1, 2, 3, 4), out_flow)

        self.assertEqual((1, 3), tuple(odd_bucket.contents()))

        self.assertEqual(("3 >= 3", "4 >= 3"), tuple(company_bucket.contents()))
Ejemplo n.º 4
0
def certain_kind_tap(data_items):
    """
    :param data_items: A sequence of unicode strings
    """
    fruit_spigot = Bucket(get_fruit)
    metal_spigot = Bucket(get_metal)

    items = stream_tap((fruit_spigot, metal_spigot), data_items)

    for batch in i_batch(100, items):
        tuple(batch)

    return fruit_spigot.contents(), metal_spigot.contents()
Ejemplo n.º 5
0
    def test_bucket_drain(self):
        """
        Test Bucket can drain it's contents.
        """
        odd_bucket = Bucket(get_odd)

        for item in self.stream:
            odd_bucket(item)

        odds = odd_bucket.drain_contents()

        self.assertEqual((1, 3), tuple(odds))

        self.assertEqual(deque(), odd_bucket.contents())
Ejemplo n.º 6
0
    def test_spigot_collects(self):
        """
        Test Spigot collects it's contents.
        """
        odd_spigot = Bucket(get_odd)

        for item in self.stream:
            odd_spigot(item)

        odds = odd_spigot.contents()

        self.assertEqual((1, 3), tuple(odds))

        self.assertNotEqual(deque(), odd_spigot.contents())