示例#1
0
 def watch(id_: str, obs: Observable):
     ingestor_mode_health = obs.pipe(
         ops.map(lambda x: self._ingestor_mode_to_health(id_, x)))
     obs.pipe(
         heartbeat(self.LIVELINESS),
         ops.map(lambda x: to_ingestor_health(id_, x)),
         self._combine_most_critical(ingestor_mode_health),
     ).subscribe(self.rmf.ingestor_health.on_next,
                 scheduler=self.scheduler)
示例#2
0
async def observable_to_async_event_generator(observable: Observable):
    queue = asyncio.Queue()

    def on_next(i):
        queue.put_nowait(i)

    observable.pipe(materialize()).subscribe(on_next=on_next)

    while True:
        value = await queue.get()
        yield value
        queue.task_done()
示例#3
0
 def watch(id_: str, obs: Observable):
     """
     :param obs: Observable[RobotState]
     """
     robot_mode_health = obs.pipe(
         ops.map(lambda x: self._robot_mode_to_health(id_, x)))
     obs.pipe(
         heartbeat(self.LIVELINESS),
         ops.map(lambda x: to_robot_health(id_, x)),
         self._combine_most_critical(robot_mode_health),
     ).subscribe(self.rmf.robot_health.on_next,
                 scheduler=self.scheduler)
示例#4
0
def accumulate_and_clear(values: Observable, clear: Observable) -> Observable:
    return merge_either(values, clear.pipe(ops.map(lambda _: True))).pipe(
        ops.scan(
            lambda collected, either: []
            if either[1] else collected + [either[0]],
            [],
        ))
示例#5
0
def generate_move_path(file_stream: rx.Observable,
                       storage_dir: str) -> rx.Observable:
    """Identify the appropriate move path for files in the file_stream."""
    return file_stream.pipe(
        operators.map(
            lambda target: fc.identify_photo_move_path(storage_dir, target)),
        operators.map(lambda target: target.clear_contents_data()),
    )
示例#6
0
    def enqueue(self,
                observable: Observable,
                group: str = 'default-group',
                retries: int = 0,
                description: str = None) -> Observable:
        def log_status(status):
            logging.debug(
                str({
                    'WorkQueue': str(self),
                    'group': group,
                    status: description
                }))

        log_status('ENQUEUED')
        output = Subject()
        errors = Subject()
        output_finalized = Subject()

        def handle_error(e, _):
            log_status('FAILED')
            errors.on_next(e)
            return empty()

        def set_output_finalized():
            output_finalized.on_next(True)

        work = of(True).pipe(
            do_action(lambda _: log_status('STARTED')),
            take_until(output_finalized),
            flat_map(lambda _: observable.pipe(
                map(lambda value: of({
                    'value': value,
                    'output': output
                })),
                retry_with_backoff(
                    retries=retries,
                    description='{}.enqueue(group={}, description={})'.format(
                        self, group, description)),
                catch(handler=handle_error), take_until(output_finalized),
                take_until_disposed())),
            concat(of(of({
                'completed': True,
                'output': output
            }))), finally_action(lambda: log_status('COMPLETED')))

        self._queue.on_next({'work': work, 'group': group})

        return output.pipe(observe_on(self.request_scheduler),
                           throw_when(errors),
                           take_while(lambda r: not r.get('completed')),
                           map(lambda r: r.get('value')),
                           finally_action(set_output_finalized))
示例#7
0
 def __init__(self, inObs: Observable, outSubject: Subject):
     """
     Parameters
     ---------
     inObs : Observable<bytes>
         Observable<bytes> that the instance subscribes to in order to receive data packets. The Observable should emit objects of type bytes
     outSubject : Subject<bytes>
         Subscribe to the outSubject to publish messages (i. e. send the to the receiver)
     """
     super().__init__()
     self._in: Observable = inObs.pipe(map(self._mapIncoming), publish())
     self._in.connect()
     self._out = Subject()
     self._out.pipe(map(self._mapOutgoing)).subscribe(outSubject)
def from_rx(
    source: rx.Observable,
    batch_size: int = None,
    overflow_strategy: OverflowStrategy = None,
    is_batched: bool = None,
) -> Flowable:
    """
    Wrap a rx.Observable and exposes it as a Flowable, relaying signals in a backpressure-aware manner.

    :param source: an rx.observable
    :param overflow_strategy: define which batches are ignored once the buffer is full
    :param batch_size: determines the number of elements that are sent in a batch
    :param is_batched: if set to True, the elements emitted by the source rx.Observable are
    either of type List or of type Iterator
    """

    if is_batched is True:
        batched_source = source
    else:
        if batch_size is None:
            batch_size = 1

        batched_source = source.pipe(operators.buffer_with_count(batch_size), )

    if isinstance(overflow_strategy, DropOld) or isinstance(
            overflow_strategy, ClearBuffer):
        return init_flowable(
            FromRxEvictingFlowable(
                batched_source=batched_source,
                overflow_strategy=overflow_strategy,
            ))

    else:
        if overflow_strategy is None:
            buffer_size = math.inf
        elif isinstance(overflow_strategy, BackPressure):
            buffer_size = overflow_strategy.buffer_size
        else:
            raise AssertionError(
                'only BackPressure is currently supported as overflow strategy'
            )

        return init_flowable(
            FromRxBufferingFlowable(
                batched_source=batched_source,
                overflow_strategy=overflow_strategy,
                buffer_size=buffer_size,
            ))
示例#9
0
    def enqueue(self,
                observable: Observable,
                group: str = None,
                retries: int = 0,
                description: str = None):
        # Provide a function returning a callable?

        description = description or str(Observable)
        key = '{}({})'.format(description, random.random())

        def log_status(status):
            logging.debug(
                str({
                    'WorkQueue': str(self),
                    'group': group,
                    'key': key,
                    status: description
                }))

        log_status('ENQUEUED')
        error: Optional[Exception] = None

        def handle_error(e):
            log_status('FAILED')
            nonlocal error
            error = e
            return of({'key': key, 'error': e})

        def throw_if_error(request):
            if error:
                return throw(error)
            else:
                return of(request)

        def extract_value(value):
            if type(value) == Observable:
                return value
            else:
                return of(value)

        request = of(True).pipe(
            do_action(lambda _: log_status('STARTED')),
            flat_map(lambda _: observable.pipe(
                flat_map(extract_value),
                map(lambda value: {
                    'key': key,
                    'value': value
                }),
                retry_with_backoff(
                    retries=retries,
                    description='{}.enqueue(group={}, description={})'.format(
                        self, group, description)),
                catch(handler=lambda e, o: handle_error(e)),
            )),
            concat(
                of({
                    'key': key,
                    'complete': True
                }).pipe(do_action(lambda _: log_status('COMPLETED')))))
        result_stream = self._output.pipe(
            filter(lambda request: request['key'] == key),
            flat_map(lambda request: throw_if_error(request)),
            take_while(lambda request: not request.get('complete')),
            flat_map(lambda request: of(request.get('value'))))
        self._requests.on_next({
            'request': request,
            'concurrency_group': group
        })
        return result_stream
示例#10
0
def right(a: Observable) -> Observable:
    return a.pipe(ops.map(lambda v: (None, v)))
示例#11
0
def left(a: Observable) -> Observable:
    return a.pipe(ops.map(lambda v: (v, None)))
示例#12
0
def generate_file_metadata(file_stream: rx.Observable) -> rx.Observable:
    """Add various file metadata to the FileTargets being streamed."""
    return file_stream.pipe(
        operators.map(fl.sha256_file),
        operators.map(fl.encode_shasum),
    )
示例#13
0
def generate_image_metadata(file_stream: rx.Observable) -> rx.Observable:
    """Add image metadata to file_stream."""
    return file_stream.pipe(
        operators.map(im.parse_image_meta_for_file_target),
        operators.map(im.identify_image_datestamp),
    )
示例#14
0
 def upstream(source: rx.Observable):
     source.pipe(operators.take_until(_stop)).subscribe(
         lambda x: buffer.append(x), on_error, on_upstream_completed)
示例#15
0
def load_file_content(file_stream: rx.Observable) -> rx.Observable:
    """Load files content from disk."""
    return file_stream.pipe(operators.map(fl.load_file_contents))
示例#16
0
 def consume(self, xs: rx.Observable):
     obs = xs.pipe(ops.do_action(lambda x: results.append(x)))
     obs.subscribe()
示例#17
0
 def forward(self, xs: rx.Observable) -> rx.Observable:
     return xs.pipe(ops.map(str))
示例#18
0
    def enqueue(self,
                observable: Observable,
                group: str = 'default-group',
                retries: int = 0,
                description: str = None):
        description = description or str(Observable)
        key = '{}({})'.format(description, random.random())

        def log_status(status):
            logging.debug(
                str({
                    'WorkQueue': str(self),
                    'group': group,
                    'key': key,
                    status: description
                }))

        log_status('ENQUEUED')
        error: Optional[Exception] = None

        def handle_error(e):
            log_status('FAILED')
            nonlocal error
            error = e
            return of({'key': key, 'error': e})

        def throw_if_error(r):
            if error:
                return throw(error)
            else:
                return of(r)

        request_disposed = Subject()

        def dispose_request():
            request_disposed.on_next(True)

        request = of(True).pipe(
            do_action(lambda _: log_status('STARTED')),
            flat_map(
                lambda _: observable.pipe(
                    map(lambda value: {
                        'key': key,
                        'value': value
                    }),
                    retry_with_backoff(retries=retries,
                                       description=
                                       '{}.enqueue(group={}, description={})'.
                                       format(self, group, description)),
                    catch(handler=lambda e, o: handle_error(e)),
                    take_until(request_disposed),
                    take_until_disposed(),
                ), ),
            concat(
                of({
                    'key': key,
                    'complete': True
                }).pipe(do_action(lambda _: log_status('COMPLETED')))),
        )
        result_stream = self._output.pipe(
            observe_on(self.request_scheduler),
            filter(lambda r: r['key'] == key),
            flat_map(lambda r: throw_if_error(r)),
            take_while(lambda r: not r.get('complete')),
            flat_map(lambda r: of(r.get('value'))),
            finally_action(dispose_request))
        self._requests.on_next({
            'request': request,
            'concurrency_group': group
        })
        return result_stream
示例#19
0
def general_node_grouper(observable: rx.Observable):
    return observable.pipe(
        op.map(lambda dic: raw_node_dict_to_formatted_node_dict(dic)),
        op.reduce(lambda acc, act: reduce_node_dict(acc, act)),
        op.map(lambda dic: node_dict_to_node_dict_with_list(dic)),
    )
示例#20
0
def general_edge_grouper(observable: rx.Observable):
    return observable.pipe(
        op.map(raw_edge_dict_to_formatted_edge_dict),
        op.reduce(reduce_edge_dict),
        op.map(edge_dict_to_edge_dict_with_list),
    )