async def demo(): one_second = timedelta(seconds=1) max_field_count = 1000 fail_every = 5 data_entry_service = InMemoryDataEntryService( field_count=max_field_count, create_interval=timedelta(milliseconds=1), fail_annotate_every=fail_every, ) annotation_service = InMemoryAnnotationService( publish_annotation=data_entry_service.annotate_field, republish_interval=one_second, fail_annotate_every=fail_every, fail_ack_every=fail_every, ) data_entry_service.start_publications( publish=annotation_service.annotate, acknowledge=annotation_service.acknowledge, interval=one_second, ) created_field_count = 0 acked_annotion_count = 0 unannotated_field_count = 0 unacked_annotion_count = 0 def consistent(_) -> bool: return (created_field_count == max_field_count and unannotated_field_count == 0 and unacked_annotion_count == 0) def inconsistent(_) -> bool: return not consistent(_) def eval_state(counter: int): nonlocal created_field_count nonlocal acked_annotion_count nonlocal unannotated_field_count nonlocal unacked_annotion_count created_field_count = data_entry_service.field_count acked_annotion_count = len(annotation_service.acknowledged_annotations) unacked_annotion_count = len( annotation_service.unacknowledged_annotations) unannotated_field_count = len(data_entry_service.unannotated_fields) print(f"Iteration: {counter + 1}") print(f"Field Count: {created_field_count}") print(f"Acked Annotation Count: {acked_annotion_count}") print(f"Unannotated Field Count: {unannotated_field_count}") print(f"Unacked Annotation Count: {unacked_annotion_count}") print("") if consistent(None): print("---------------") print("Eventually has arrived :)") print("---------------") sched = rx.interval(timedelta(seconds=1)) composed = sched.pipe(take_while(inconsistent)) composed.subscribe(eval_state)
def collect(messages: Observable[IbApiMessage]) -> Observable[List[Position]]: return messages.pipe( _.filter(lambda m: _is_position(m) or _is_position_end(m)), _.take_while(lambda m: not _is_position_end(m)), _.map(_unpack_position), _.reduce(lambda positions, position: [*positions, position], []), )
def light_state_equals(light_id: str, light_final_state: str, test_context: Data.TestContext, loop: asyncio.AbstractEventLoop, awaitables: List[RxObservable]): assert isinstance(light_id, str) assert isinstance(light_final_state, str) def take_while_state(payload: Structs.s_lights_state) -> bool: return payload.newState != light_final_state timeout_sec = 10.0 lightbulb: Data.Lightbulb = test_context.lightbulbs[light_id] observable: RxObservable = lightbulb.light_state.pipe( RxOp.timeout(timeout_sec), RxOp.observe_on(scheduler=AsyncIOScheduler(loop)), RxOp.take_while(take_while_state, inclusive=True), ) observable.subscribe(on_next=lambda i: print(f"on_next: {i}"), on_error=lambda e: print(f"on_error: {e}"), on_completed=lambda: print("on_completed"), scheduler=AsyncIOScheduler(loop)) awaitables.append(observable)
def factory(): def predicate(x): nonlocal invoked invoked += 1 return is_prime(x) return xs.pipe(ops.take_while(predicate))
def _solve(print=print): total = rx.from_iterable(primes()) \ .pipe( ops.take_while(lambda p: p < 2000000), ops.sum(), ).run() print(f'The sum of primes below 2m: {total}') return True
def collect(messages: Observable[IbApiMessage], request_id: int) -> Observable[AccountSummary]: return messages.pipe( _.filter( lambda m: _is_account_summary(m) or _is_account_summary_end(m)), _.filter(lambda m: _request_id(m) == request_id), _.take_while(lambda m: not _is_account_summary_end(m)), _.map(_unpack_account_summary), _.reduce(lambda summary, data: _add_data_to_summary(data, summary), AccountSummary()))
def __init__(self, client: MQTTClientWrapper, name="ds18", config=None): super().__init__(client, None, config=config, name=name) self.topic_base = self.config["ds18"]["topicBase"] self.topic = self.topic_base + "/temperature/+" self.can_subscribe.pipe( ops.take_while( lambda x: not x ), # while cannot subscribe, continue to take (should not happen) ).subscribe(lambda x: self.subscribe_on_broker(self.topic)) self.sensors = self.config["ds18"]["sensors"]
def factory(): def predicate(x): nonlocal invoked invoked += 1 if invoked == 3: raise Exception(ex) return is_prime(x) return xs.pipe(ops.take_while(predicate))
def collect( messages: Observable[IbApiMessage], request_id: int, data_type: types.HistoricalDataType) -> Observable[List[types.BarData]]: return messages.pipe( _.filter( lambda m: _is_historical_data(m) or _is_historical_data_end(m)), _.filter(lambda m: _request_id(m) == request_id), _.take_while(lambda m: not _is_historical_data_end(m)), _.map(lambda m: _unpack_historical_data(m, data_type)), _.reduce(lambda bars, bar: [*bars, bar], []), )
def download(destination_file): do_action(lambda _: os.makedirs(destination_file, exist_ok=True)) return of(create_downloader(destination_file)).pipe( flat_map(lambda downloader: forever().pipe( map(lambda _: next_chunk(downloader)), take_while(lambda progress: progress < 1, inclusive=True), map( lambda progress: { 'progress': progress, 'downloaded_bytes': int(int(file['size']) * progress), 'file': file }))))
def enqueue(self, observable: Observable, group: str = 'default-group', retries: int = 0, description: str = None) -> Observable: def log_status(status): logging.debug( str({ 'WorkQueue': str(self), 'group': group, status: description })) log_status('ENQUEUED') output = Subject() errors = Subject() output_finalized = Subject() def handle_error(e, _): log_status('FAILED') errors.on_next(e) return empty() def set_output_finalized(): output_finalized.on_next(True) work = of(True).pipe( do_action(lambda _: log_status('STARTED')), take_until(output_finalized), flat_map(lambda _: observable.pipe( map(lambda value: of({ 'value': value, 'output': output })), retry_with_backoff( retries=retries, description='{}.enqueue(group={}, description={})'.format( self, group, description)), catch(handler=handle_error), take_until(output_finalized), take_until_disposed())), concat(of(of({ 'completed': True, 'output': output }))), finally_action(lambda: log_status('COMPLETED'))) self._queue.on_next({'work': work, 'group': group}) return output.pipe(observe_on(self.request_scheduler), throw_when(errors), take_while(lambda r: not r.get('completed')), map(lambda r: r.get('value')), finally_action(set_output_finalized))
def run(onComplete=lambda: None): limit = 10 ** 22 optimal_thread_count = multiprocessing.cpu_count() pool_scheduler = ThreadPoolScheduler(optimal_thread_count) count = rx.from_iterable(pandigital_step_numbers()) \ .pipe( ops.take_while(lambda n: n < limit), ops.count(), ) \ .run() onComplete(count)
def monitor(): def is_running(state): return state in [Task.State.UNSUBMITTED, Task.State.READY, Task.State.RUNNING] return interval(_MONITORING_FREQUENCY).pipe( flat_map(lambda _: execute( action=load_status, description='monitor task ' + str(task)) ), flat_map(extract_state), distinct_until_changed(), take_while(is_running, inclusive=True) )
def get_pipeline(seed: int, name: str, print_output=False) -> rx.pipe: """Returns an rx pipeline that parses the raw output from MCERD into dictionaries. Each dictionary contains the same keys. If certain value cannot be parsed from the output (i.e. the raw line does not contain it), either the value from the previous dictionary is carried over or a default value is used. Args: seed: seed used in the MCERD process name: name of the process (usually the name of the recoil element) print_output: whether output is printed to console """ # TODO add handling for fatal error messages return rx.pipe( ops.map(lambda x: x.strip()), MCERD._conditional_printer( print_output, f"simulation process with seed {seed}."), observing.reduce_while( reducer=str_reducer, start_from=lambda x: x == MCERD._INIT_STARTS, end_at=lambda x: x == MCERD._INIT_ENDS), observing.reduce_while( reducer=str_reducer, start_from=lambda x: x.startswith(MCERD._FINAL_STARTS), end_at=lambda x: x.startswith(MCERD._FINAL_ENDS)), ops.scan(lambda acc, x: { MCERD.PRESIM: acc[MCERD.PRESIM] and x != MCERD.PRESIM_FINISHED, **parse_raw_output(x, end_at=lambda y: y.startswith(MCERD._FINAL_STARTS)) }, seed={MCERD.PRESIM: True}), ops.scan(lambda acc, x: dict_accumulator(acc, x, default={ MCERD.SEED: seed, MCERD.NAME: name, MCERD.MSG: "", MCERD.IS_RUNNING: True }), seed={ MCERD.CALCULATED: 0, MCERD.TOTAL: 0, MCERD.PERCENTAGE: 0 }), ops.take_while(lambda x: x[MCERD.IS_RUNNING], inclusive=True))
def subscriptionFunction(observer, scheduler): message = RxImpMessage(topic, 0, RxImpMessage.STATE_SUBSCRIBE, json.dumps(payload)) publisher: Subject = Subject() lock = Lock() currentCount = 0 queue = [] def orderingSubscriber(msg: RxImpMessage): nonlocal currentCount nonlocal queue with lock: currentCount += 1 queue.append(msg) queue.sort(key=lambda x: x.count) toNext = [msg for msg in queue if msg.count < currentCount] queue = [msg for msg in queue if msg.count >= currentCount] for msg in toNext: publisher.on_next(msg) def isRelevant(msg: RxImpMessage): return msg.rx_state == RxImpMessage.STATE_COMPLETE or msg.rx_state == RxImpMessage.STATE_ERROR or msg.rx_state == RxImpMessage.STATE_NEXT secondSubscription: Disposable = self._in.pipe( filter(lambda x: x.id == message.id), filter(lambda x: isRelevant(x)), map(lambda x: self._checkError(x)), ).subscribe(on_next=lambda x: orderingSubscriber(x), on_error=lambda err: publisher.on_error(err)) subscription: Disposable = publisher.pipe( take_while(lambda x: self._checkNotComplete(x)), map(lambda x: json.loads(x.payload)), ).subscribe(observer) self._out.on_next(message) def signalUnsubscribe(): msg = RxImpMessage(message.topic, 0, RxImpMessage.STATE_DISPOSE, None, id=message.id) secondSubscription.dispose() subscription.dispose() self._out.on_next(msg) return lambda: signalUnsubscribe()
def download_from_drive(destination_file): def create_downloader(): request = get_service(credentials).files().get_media( fileId=f['id']) return MediaIoBaseDownload(fd=destination_file, request=request, chunksize=CHUNK_SIZE) downloader = create_downloader() return forever().pipe( map(lambda _: next_chunk(downloader)), take_while(lambda progress: progress < 1, inclusive=True), map( lambda progress: { 'downloaded_bytes': int(total_bytes * progress), 'total_bytes': total_bytes, 'file': f }))
def download_from_drive(destination_file): def create_downloader(): request = get_service(credentials).files().get_media( fileId=f['id']) return MediaIoBaseDownload(fd=destination_file, request=request, chunksize=CHUNK_SIZE) downloader = create_downloader() return forever().pipe( map(lambda _: next_chunk(downloader)), take_while(lambda p: p < 1, inclusive=True), flat_map(lambda p: progress( default_message= 'Downloaded {downloaded_files} of {total_files} files ({downloaded} of {total})', message_key='tasks.drive.download_folder', downloaded_bytes=int(total_bytes * p), downloaded=format_bytes(int(total_bytes * p)), total_bytes=total_bytes, total=format_bytes(total_bytes), file=f)))
def running_check(process: subprocess.Popen, first_check: float, interval: float) -> rx.Observable: """Periodically checks if the given process is running. Args: process: process to be monitored first_check: seconds until the first check interval: interval between each check Return: rx.Observable that fires dictionaries after each check """ return rx.timer(first_check, interval).pipe( # TODO change this to run at an increasing interval, i.e: # - first check after 0.0 seconds, # - second check after 0.2 seconds, # - third after 1.0, ... etc. # MCERD is likely to crash early (?) so it makes sense to # run the check more frequently at the beginning. ops.map(lambda _: {MCERD.IS_RUNNING: MCERD.is_running(process)}), ops.take_while(lambda x: x[MCERD.IS_RUNNING], inclusive=True))
op.catch() op.retry() """Utility""" op.delay() op.materialize() op.time_interval() op.timeout() op.timestamp() """Conditional and Boolean""" op.all() op.contains() op.default_if_empty() op.sequence_equal() op.skip_until() op.skip_while() op.take_until() op.take_while() """Connectable""" op.publish() op.ref_count() op.replay() """Combining""" op.combine_latest() op.merge() op.start_with() op.zip()
import rx import rx.operators as ops numbers = rx.from_([1, 2, 3, 4]) numbers.pipe(ops.take_while(lambda i: i < 2)).subscribe( on_next=lambda i: print("on_next {}".format(i)), on_error=lambda e: print("on_error: {}".format(e)), on_completed=lambda: print("on_completed"))
import rx from rx import operators as ops rx.from_( [2, 5, 21, 5, 2, 1, 5, 63, 127, 12] ).pipe( ops.take_while(lambda i: i < 100) ).subscribe( on_next=lambda i: print(i), on_completed=lambda: print("Done!") )
def run(self, print_output=True, ct: Optional[CancellationToken] = None, poll_interval=10, first_check=0.2, max_time=None, ct_check=0.2) -> rx.Observable: """Starts the MCERD process. Args: print_output: whether MCERD output is also printed to console ct: token that is checked periodically to see if the simulation should be stopped. poll_interval: seconds between each check to see if the simulation process is still running. first_check: seconds until the first time mcerd is polled. max_time: maximum running time in seconds. ct_check: how often cancellation is checked in seconds. Return: observable stream where each item is a dictionary. All dictionaries contain the same keys. """ # Create files necessary to run MCERD self.create_mcerd_files() cmd = self.get_command() ct = ct or CancellationToken() process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=gf.get_bin_dir(), universal_newlines=True) errs = rx.from_iterable(iter(process.stderr.readline, "")) outs = rx.from_iterable(iter(process.stdout.readline, "")) is_running = MCERD.running_check(process, first_check, poll_interval) ct_check = MCERD.cancellation_check(process, ct_check, ct) if max_time is not None: timeout = MCERD.timeout_check(process, max_time, ct) else: timeout = rx.empty() thread_count = multiprocessing.cpu_count() pool_scheduler = ThreadPoolScheduler(thread_count) merged = rx.merge(errs, outs).pipe( ops.subscribe_on(pool_scheduler), MCERD.get_pipeline(self._seed, self._rec_filename, print_output=print_output), ops.combine_latest(rx.merge(is_running, ct_check, timeout)), ops.starmap( lambda x, y: { **x, **y, MCERD.IS_RUNNING: x[MCERD.IS_RUNNING] and y[MCERD.IS_RUNNING] }), ops.take_while(lambda x: x[MCERD.IS_RUNNING], inclusive=True), ) # on_completed does not get called if the take_while condition is # inclusive so this is a quick fix to get the files deleted. # TODO surely there is a way to get the on_completed called? def del_if_not_running(x): if not x[MCERD.IS_RUNNING]: self.delete_unneeded_files() return merged.pipe( ops.do_action(on_next=del_if_not_running, on_error=lambda _: self.delete_unneeded_files(), on_completed=self.delete_unneeded_files))
def enqueue(self, observable: Observable, group: str = 'default-group', retries: int = 0, description: str = None): description = description or str(Observable) key = '{}({})'.format(description, random.random()) def log_status(status): logging.debug( str({ 'WorkQueue': str(self), 'group': group, 'key': key, status: description })) log_status('ENQUEUED') error: Optional[Exception] = None def handle_error(e): log_status('FAILED') nonlocal error error = e return of({'key': key, 'error': e}) def throw_if_error(r): if error: return throw(error) else: return of(r) request_disposed = Subject() def dispose_request(): request_disposed.on_next(True) request = of(True).pipe( do_action(lambda _: log_status('STARTED')), flat_map( lambda _: observable.pipe( map(lambda value: { 'key': key, 'value': value }), retry_with_backoff(retries=retries, description= '{}.enqueue(group={}, description={})'. format(self, group, description)), catch(handler=lambda e, o: handle_error(e)), take_until(request_disposed), take_until_disposed(), ), ), concat( of({ 'key': key, 'complete': True }).pipe(do_action(lambda _: log_status('COMPLETED')))), ) result_stream = self._output.pipe( observe_on(self.request_scheduler), filter(lambda r: r['key'] == key), flat_map(lambda r: throw_if_error(r)), take_while(lambda r: not r.get('complete')), flat_map(lambda r: of(r.get('value'))), finally_action(dispose_request)) self._requests.on_next({ 'request': request, 'concurrency_group': group }) return result_stream
def enqueue(self, observable: Observable, group: str = None, retries: int = 0, description: str = None): # Provide a function returning a callable? description = description or str(Observable) key = '{}({})'.format(description, random.random()) def log_status(status): logging.debug( str({ 'WorkQueue': str(self), 'group': group, 'key': key, status: description })) log_status('ENQUEUED') error: Optional[Exception] = None def handle_error(e): log_status('FAILED') nonlocal error error = e return of({'key': key, 'error': e}) def throw_if_error(request): if error: return throw(error) else: return of(request) def extract_value(value): if type(value) == Observable: return value else: return of(value) request = of(True).pipe( do_action(lambda _: log_status('STARTED')), flat_map(lambda _: observable.pipe( flat_map(extract_value), map(lambda value: { 'key': key, 'value': value }), retry_with_backoff( retries=retries, description='{}.enqueue(group={}, description={})'.format( self, group, description)), catch(handler=lambda e, o: handle_error(e)), )), concat( of({ 'key': key, 'complete': True }).pipe(do_action(lambda _: log_status('COMPLETED'))))) result_stream = self._output.pipe( filter(lambda request: request['key'] == key), flat_map(lambda request: throw_if_error(request)), take_while(lambda request: not request.get('complete')), flat_map(lambda request: of(request.get('value')))) self._requests.on_next({ 'request': request, 'concurrency_group': group }) return result_stream