Esempio n. 1
0
def consume_stream(target, input_dir=None):
    """Read lines from the standard input or files in a directory.

    Behaves as a generator, should receive a .send() call to send a
    line to the target.

    """
    file_names = get_file_names(input_dir) if input_dir else []

    with contextlib.closing(
            fileinput.FileInput(file_names,
                                openhook=fileinput.hook_compressed)) as lines:
        targets = [target] if target is not None else []
        with consumers.closing(*targets):

            for line in lines:

                if not line.strip():
                    continue

                if isinstance(line, bytes):
                    line = line.decode('utf-8')

                if target is not None:
                    result = target.send(line)
                    if result is not consumers.SendNext:
                        yield result
                else:
                    yield line
Esempio n. 2
0
def consume_stream(target, input_dir=None):
    """Read lines from the standard input or files in a directory.

    Behaves as a generator, should receive a .send() call to send a
    line to the target.

    """
    file_names = get_file_names(input_dir) if input_dir else []

    with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines:
        targets = [target] if target is not None else []
        with consumers.closing(*targets):

            for line in lines:
                if not line.strip():
                    continue

                if isinstance(line, bytes):
                    line = line.decode('utf-8')

                if target is not None:
                    result = target.send(line)
                    if result is not consumers.SendNext:
                        yield result
                else:
                    yield line
Esempio n. 3
0
def _conductor(merger, tagged_sources):
    result = None

    with closing(merger):
        # The first step: push each target once.
        for source in tagged_sources.values():
            try:
                result = next(source)
                if result is None:
                    yield
            except StopIteration:
                pass

        item, source_priority = result

        # The second step: push targets as merge requests.
        while True:
            yield item, source_priority

            while source_priority:
                tag = source_priority.pop(0)
                try:
                    item, source_priority = next(tagged_sources[tag])
                except StopIteration:
                    pass
                else:
                    break
            else:
                return
Esempio n. 4
0
def _conductor(merger, tagged_sources):
    result = None

    with closing(merger):
        # The first step: push each target once.
        for source in tagged_sources.values():
            try:
                result = next(source)
                if result is None:
                    yield
            except StopIteration:
                pass

        item, source_priority = result

        # The second step: push targets as merge requests.
        while True:
            yield item, source_priority

            while source_priority:
                tag = source_priority.pop(0)
                try:
                    item, source_priority = next(tagged_sources[tag])
                except StopIteration:
                    pass
                else:
                    break
            else:
                return
Esempio n. 5
0
 def SpecialTarget(target):
     try:
         with consumers.closing(target):
             while True:
                 item = yield
                 target.send(item)
     except ThrownException:
         pass
Esempio n. 6
0
 def SpecialTarget(target):
     try:
         with consumers.closing(target):
             while True:
                 item = yield
                 target.send(item)
     except ThrownException:
         pass
Esempio n. 7
0
def from_simple_queue(target, queue):
    with consumers.closing(target):
        while True:
            item = queue.get()

            if item is StopIteration:
                break

            target.send(item)
Esempio n. 8
0
def from_simple_queue(target, queue):
    with consumers.closing(target):
        while True:
            item = queue.get()

            if item is StopIteration:
                break

            target.send(item)
Esempio n. 9
0
def _merger(tagged_targets, provider=None):
    if provider is None:

        def provider(current):
            return current

    targets = set(tagged_targets.values())
    with closing(*targets):

        items = []
        heapq.heapify(items)

        applied_sources = set()
        closed_sources = set()
        all_sources = set(tagged_targets)

        try:
            # First step: get at least one item from each source
            while applied_sources != all_sources:
                try:
                    tag, item = yield
                except TaggedGeneratorExit as e:
                    applied_sources.add(e.tag)
                    closed_sources.add(e.tag)
                else:
                    heapq.heappush(items, (provider(item), tag, item))
                    applied_sources.add(tag)

            # Second step: push forward the smallest item to its target
            # and push back to conductor the priorities of the sources.
            while closed_sources != applied_sources:
                priorities = [i[1] for i in sorted(items)]

                _, tag, item = heapq.heappop(items)

                result = tagged_targets[tag].send(item)

                try:
                    tag, item = yield result, priorities
                except TaggedGeneratorExit as e:
                    closed_sources.add(e.tag)
                    logger.debug("%s is closed", tag)
                else:
                    heapq.heappush(items, (provider(item), tag, item))

        finally:
            for _, tag, item in sorted(items):
                tagged_targets[tag].send(item)
Esempio n. 10
0
def _merger(tagged_targets, provider=None):
    if provider is None:

        def provider(current):
            return current

    targets = set(tagged_targets.values())
    with closing(*targets):

        items = []
        heapq.heapify(items)

        applied_sources = set()
        closed_sources = set()
        all_sources = set(tagged_targets)

        try:
            # First step: get at least one item from each source
            while applied_sources != all_sources:
                try:
                    tag, item = yield
                except TaggedGeneratorExit as e:
                    applied_sources.add(e.tag)
                    closed_sources.add(e.tag)
                else:
                    heapq.heappush(items, (provider(item), tag, item))
                    applied_sources.add(tag)

            # Second step: push forward the smallest item to its target
            # and push back to conductor the priorities of the sources.
            while closed_sources != applied_sources:
                priorities = [i[1] for i in sorted(items)]

                _, tag, item = heapq.heappop(items)

                result = tagged_targets[tag].send(item)

                try:
                    tag, item = yield result, priorities
                except TaggedGeneratorExit as e:
                    closed_sources.add(e.tag)
                    logger.debug('%s is closed', tag)
                else:
                    heapq.heappush(items, (provider(item), tag, item))

        finally:
            for _, tag, item in sorted(items):
                tagged_targets[tag].send(item)
Esempio n. 11
0
def consume_stream(target, input_dir=None):
    """Read lines from the standard input or files in a directory.

    Behaves as a generator, should receive a .send() call to send a
    line to the target.

    """
    file_names = get_file_names(input_dir) if input_dir else []

    with contextlib.closing(fileinput.FileInput(file_names, openhook=fileinput.hook_compressed)) as lines:
        with consumers.closing(target):

            for line in lines:
                result = target.send(line)

                if result is not consumers.SendNext:
                    yield result
Esempio n. 12
0
def consume_stream(target, input_dir=None):
    """Read lines from the standard input or files in a directory.

    Behaves as a generator, should receive a .send() call to send a
    line to the target.

    """
    file_names = get_file_names(input_dir) if input_dir else []

    with contextlib.closing(
            fileinput.FileInput(file_names,
                                openhook=fileinput.hook_compressed)) as lines:
        with consumers.closing(target):

            for line in lines:
                result = target.send(line)

                if result is not consumers.SendNext:
                    yield result