def _runjob(pipe: Connection, procs: TaskCacheList, keys: List[int]): "runs a job" if pipe.poll(): return frame = next(iter(procs.run()), None) if frame is None: pipe.send((None, None)) return if callable(getattr(frame, 'bead', None)): raise NotImplementedError() for i in keys: if pipe.poll(): return try: out = (i, frame[i]) except ProcessorException as exc: out = (i, exc) if pipe.poll(): return pipe.send(out) if not pipe.poll(): pipe.send((None, None))
def _multiprocess_generator_pool_child(pipe: Connection) -> None: while True: # wait for a message pipe.poll(None) # read the message msg = pipe.recv() if msg == SENTINEL: # received a sentinel message to terminate self break else: # start of a new function invocation func, batchsize, kwargs = msg # start a fresh batch of results batch = [] try: for result in func(**kwargs): batch.append(result) # batch is full, send it and start a new one if len(batch) >= batchsize: pipe.send([kwargs, batch, None]) batch = [] except Exception as e: # if an error happened send it up pipe.send([kwargs, batch, e]) # continue to the next arg else: # no error was thrown # send any leftover lines smaller than a batch pipe.send([kwargs, batch, None]) # send a sentinel to say we've finished an arg pipe.send(SENTINEL)
def run_daemon(qa_pipe: Connection): wiki_daemon = WikiDaemon(WIKI_PAGE) wiki_daemon.update_wiki_cache() wiki_daemon.reload_spacy_docs() print("wiki_daemon: Child process started") next_wiki_update = time.time() + UPDATE_PERIOD_SECS while True: now = time.time() if next_wiki_update < now: print("wiki_daemon: Checking Wikipedia for updates") updated = wiki_daemon.update_wiki_cache() if updated: print("wiki_daemon: Got new revision, updating") next_wiki_update = now + UPDATE_PERIOD_SECS if qa_pipe.poll(timeout=next_wiki_update - now): try: question = qa_pipe.recv() qa_pipe.send(wiki_daemon.inquiry(question)) except EOFError: # Pipe was closed on other end, we're done here qa_pipe.close() return except ValueError: print("Answer was too large to send!") # Make sure the caller isn't still waiting for an object try: qa_pipe.send("") except EOFError: qa_pipe.close() return
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters): return False if cls.ASYNCHRONOUS: cls.enter_async_send_mode(send_config.get_data_to_send) else: cls.prepare_sync_send(ctrl_connection) exit_requested = False buffer_size = cls.CONTINUOUS_SEND_BUFFER_SIZE if send_config.continuous else cls.SEND_BUFFER_SIZE if not cls.ASYNCHRONOUS and buffer_size == 0: logger.warning("Send buffer size is zero!") while not exit_requested and not send_config.sending_is_finished(): if cls.ASYNCHRONOUS: time.sleep(0.5) else: cls.send_sync(send_config.get_data_to_send(buffer_size)) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True) if result == cls.Command.STOP.name: exit_requested = True break if exit_requested: logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__)) if send_config.sending_is_finished(): logger.debug("{}: sending is finished.".format(cls.__class__.__name__)) cls.shutdown_device(ctrl_connection) ctrl_connection.close()
def update(self, conn: Connection, rtsp_url: str, buffer: bool): """ Runs the rtspcam thread to grab data and keep the buffer empty. :param conn: the Pipe to transmit data :param rtsp_url: the url of the rtspcam. :param buffer: should the client read frame by frame from the buffer or just grab the latest frame? :return: """ self.log.info(f"Starting video capture client for {rtsp_url}") cap = VideoCapture(rtsp_url, CAP_FFMPEG) self.log.info("Started...") run = True while run: if not conn.poll() and not buffer: self.log.debug("clearing buffer frame") cap.grab() continue rec_dat = conn.recv() if rec_dat == self.SEND_FRAME: self.log.debug("Sending next frame to parent process") return_value, frame = cap.read() conn.send(frame) elif rec_dat == self.END_PROCESS: self.log.debug("Closing connection") cap.release() run = False self.log.info("Camera Connection Closed") conn.close()
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters): return False try: cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name]) except NotImplementedError: # Many SDRs like HackRF or AirSpy do not need to calculate READ_SAMPLES # as default values are either fine or given by the hardware pass if cls.ASYNCHRONOUS: cls.enter_async_receive_mode(data_connection) else: cls.prepare_sync_receive(ctrl_connection) exit_requested = False while not exit_requested: if cls.ASYNCHRONOUS: time.sleep(0.5) else: cls.receive_sync(data_connection) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False) if result == cls.Command.STOP.name: exit_requested = True break cls.shutdown_device(ctrl_connection) data_connection.close() ctrl_connection.close()
def lulapy_broker(conn: Connection) -> None: connections.append(conn) funcs: Dict[Text, Callable] = { 'message': message, 'subscribe': subscribe, 'unsubscribe': unsubscribe, 'close': close, 'add_connection': add_connection } while True: if len(connections) == 0: break for conn in connections: if conn.closed: connections.remove(conn) for topic in topics.keys(): topics[topic].remove(conn) elif conn.poll(): data = conn.recv() _type = data['type'] funcs[_type](data, conn)
def relay( sub_id: str, shutdown: threading.Event, channel: connection.Connection, callback: Callable[[Dict[str, Any]], None], ) -> None: while not shutdown.is_set(): try: if channel.poll(timeout=1): ev = channel.recv() if ev['event_name'] == eventNames.SUBSCRIBED: logger.info( 'Subscriber#{0} subscribe ack received'.format( sub_id, ), ) elif ev['event_name'] == eventNames.UNSUBSCRIBED: logger.info( 'Subscriber#{0} unsubscribe ack received'.format( sub_id, ), ) break elif ev['event_name'] == eventNames.DISPATCHER_SHUTDOWN: logger.info( 'Subscriber#{0} received dispatcher shutdown event' .format(sub_id, ), ) break else: callback(ev) except queue.Empty: pass except EOFError: break except KeyboardInterrupt: break logger.debug('bbye!!!')
def get_last_message(connection: Connection) -> dict: message = {} try: while connection.poll(): message = connection.recv() except EOFError: pass return message
def Pconsumer_fcn(Pconn: Connection, loopCondition: Value): counter = 0 while loopCondition.value == 1: if Pconn.poll(0.1): _ = Pconn.recv() counter += 1 Pconn.close() print(f"Consumer msg: Received: {counter} messages\n")
def show_feed(index: int, pipe: Connection): name = "Cam " + str(index) size = big print("running size:", size) cap = cv2.VideoCapture(index, backend) cap.set(cv2.CAP_PROP_FRAME_WIDTH, size[0]) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, size[1]) save_dir = "C:/users/phill/companion app save folder/" vid_ext = ".avi" codec = "DIVX" frame_size = big fps = 30 writer = cv2.VideoWriter() self.writer = cv2.VideoWriter( save_dir + timestamp + self.name + '_output' + vid_ext, cv2.VideoWriter_fourcc(*codec), fps, frame_size) start = time.time() num_frames = 0 resize = 0 while True: if pipe.poll(): msg = pipe.recv() if msg == "small": resize = 1 elif msg == "big": resize = 2 elif msg == "reset": resize = 0 elif msg == "close": cv2.destroyWindow(name) cap.release() break ret, frame = cap.read() if ret and frame is not None: if resize == 1: frame = imutils.resize(frame, width=small[0]) elif resize == 2: frame = imutils.resize(frame, width=big[0]) cv2.imshow(name, frame) num_frames += 1 else: print("Cam failed") cv2.destroyWindow(name) cap.release() break keypress = cv2.waitKey(1) % 0xFF if keypress == ord('q'): break end = time.time() seconds = end - start if seconds > 0: fps = num_frames / seconds else: fps = 0 print("time taken:", seconds, "fps:", fps, "camera:", index, "frames handled:", num_frames) cap.release()
def _run_job(pipe_to_parent: Connection, job: MountainJob) -> None: result0 = job._execute(print_console_out=False) pipe_to_parent.send(result0.getObject()) # wait for message to return while True: if pipe_to_parent.poll(): pipe_to_parent.recv() return time.sleep(0.1)
def _pjh_run_job(pipe_to_parent: Connection, job: Dict[str, Any], kachery_config: dict) -> None: import kachery as ka ka.set_config(**kachery_config) hither._run_job(job) pipe_to_parent.send(job['result'].serialize()) # wait for message to return while True: if pipe_to_parent.poll(): pipe_to_parent.recv() return time.sleep(0.1)
def traffic_gen(mac: str, pipe_rcv: connection.Connection) -> None: with configure_eth_if() as so: payload = bytes.fromhex('ff') # DUMMY_TRAFFIC code payload += bytes(1485) eth_frame = Ether(dst=mac, src=so.getsockname()[4], type=0x2222) / raw(payload) try: while pipe_rcv.poll() is not True: so.send(raw(eth_frame)) except Exception as e: raise e
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict): if not cls.init_device( ctrl_connection, is_tx=True, parameters=dev_parameters): ctrl_connection.send("failed to start tx mode") return False if cls.ASYNCHRONOUS: ret = cls.enter_async_send_mode(send_config.get_data_to_send) else: ret = cls.prepare_sync_send(ctrl_connection) if ret != 0: ctrl_connection.send("failed to start tx mode") return False exit_requested = False buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE if not cls.ASYNCHRONOUS and buffer_size == 0: logger.warning("Send buffer size is zero!") ctrl_connection.send("successfully started tx mode") while not exit_requested and not send_config.sending_is_finished(): if cls.ASYNCHRONOUS: try: time.sleep(0.5) except KeyboardInterrupt: pass else: cls.send_sync(send_config.get_data_to_send(buffer_size)) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True) if result == cls.Command.STOP.name: exit_requested = True break if not cls.ASYNCHRONOUS: # Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure # that the send buffer on the SDR is cleared time.sleep(0.75) if exit_requested: logger.debug("{}: exit requested. Stopping sending".format( cls.__class__.__name__)) if send_config.sending_is_finished(): logger.debug("{}: sending is finished.".format( cls.__class__.__name__)) cls.shutdown_device(ctrl_connection, is_tx=True) ctrl_connection.close()
def receive(conn: Connection, _pipeline: Callable[[Iterator[dict]], Iterator]): """ Процедура получения строки через сетевое соединение и обратной передачи данных нормализации """ sentence = '' stemming.init_cache() tokenization.init_cache() config.init_cache() try: with stemming.jstem_ctx() as stemmer: while True: if conn.poll(timeout=_RTN_CONNECTION_LIFE_TIME): sentence = conn.recv() logger.debug(f'Received: "{sentence}"') if _PROFILER: profiler.enable() start = process_time() try: analisys = normalization.analyze(sentence, stemmer) result = list(_pipeline(analisys)) except Exception as e: logger.error(f'Normalization failed with error: {e} \n {sentence}') result = [] end = process_time() if _PROFILER: profiler.disable() conn.send(result) logger.debug(f'Sent: {result}') perflog.debug(f'RTN time: {round((end-start) * 1000, 2)} ms') if _PROFILER: profiler.dump_stats(os.path.join(settings.ROOT_PATH, f'rtn_{time()}.cprof')) else: raise TimeoutError except EOFError: logger.info('Incoming connection closed') except TimeoutError: logger.warning('Incoming connection timeout') except Exception: logger.exception(sentence if sentence else 'Normalization Failed') finally: stemming.cache_clear() tokenization.cache_clear() config.cache_clear() logger.debug('Closing connection...') conn.close() logger.debug('Connection closed')
def func_wrapper(create_callback: Callable, conn: Connection, *args: Any) -> None: """Wrap a function with connection to receive and send data.""" running = True # pylint: disable=unused-argument def handle_signal(signum: int, frame: Any) -> None: """Handle signal.""" nonlocal running running = False conn.close() signal.signal(signal.SIGTERM, handle_signal) signal.signal(signal.SIGINT, handle_signal) try: callback = create_callback(*args) except Exception as exc: # pylint: disable=broad-except LOGGER.error("Failed to create callback: %s", exc) return while running: while running: if conn.poll(): break sleep(0.5) try: data = conn.recv() except EOFError: LOGGER.debug("Nothing more to receive") break except OSError: LOGGER.debug("Connection is closed") break try: result = callback(data) except Exception as exc: # pylint: disable=broad-except LOGGER.error("Failed to run callback: %s", exc) break if not running: break try: conn.send(result) except ValueError: LOGGER.error("Failed to send result %s", result) except OSError: LOGGER.debug("Connection is closed") break LOGGER.debug("Exiting process")
def my_first_proc(conn: Connection, some_list: list): some_list[0] = 999 print("Proc 1 id:", id(some_list), some_list, "pid:", os.getpid()) for _ in range(10): print("🐉") time.sleep(1) while not conn.poll(): time.sleep(1) received_list = conn.recv() print("Proc 1 received from proc 2, id:", id(received_list))
def _DecompileFileProcess (inputPipe: connection.Connection, outputPipe: connection.Connection) -> None: """ Intended to run in a separate process don't call this directly. """ while True: inputPipe.poll(None) request = inputPipe.recv() # type: dict targetFilePath = request["targetFilePath"] # type: str destinationFilePath = request["destinationFilePath"] # type: str printFileName = request["printFileName"] # type: str if printFileName is None: printFileName = targetFilePath try: with open(destinationFilePath, "w+", encoding = "utf-8") as destinationFile: destinationFile.write(str(unpyc3.decompile(targetFilePath))) except Exception as e: # noinspection SpellCheckingInspection print("Failed to decompile '", printFileName + "' with 'unpyc3' trying alternative 'decompyle3'. \n" + str(e), file = sys.stderr) try: with open(destinationFilePath, "w+", encoding = "utf-8") as destinationFile: Decompyle3Main.decompile_file(targetFilePath, outstream = destinationFile) except Exception as e: # noinspection SpellCheckingInspection print("decompyle3 failed to decompile '", printFileName + "'. \n" + str(e), file = sys.stderr) outputPipe.send(False) continue outputPipe.send(True) continue
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters): ctrl_connection.send("failed to start tx mode") return False if cls.ASYNCHRONOUS: ret = cls.enter_async_send_mode(send_config.get_data_to_send) else: ret = cls.prepare_sync_send(ctrl_connection) if ret != 0: ctrl_connection.send("failed to start tx mode") return False exit_requested = False buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE if not cls.ASYNCHRONOUS and buffer_size == 0: logger.warning("Send buffer size is zero!") ctrl_connection.send("successfully started tx mode") while not exit_requested and not send_config.sending_is_finished(): if cls.ASYNCHRONOUS: try: time.sleep(0.5) except KeyboardInterrupt: pass else: cls.send_sync(send_config.get_data_to_send(buffer_size)) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True) if result == cls.Command.STOP.name: exit_requested = True break if not cls.ASYNCHRONOUS: # Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure # that the send buffer on the SDR is cleared time.sleep(0.75) if exit_requested: logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__)) if send_config.sending_is_finished(): logger.debug("{}: sending is finished.".format(cls.__class__.__name__)) cls.shutdown_device(ctrl_connection, is_tx=True) ctrl_connection.close()
def log_timer(receiver: Connection, sender: Connection): print('downloading ', end='', flush=True) sender.close() # start ticker self.ticker.start() while True: if receiver.poll(): done = receiver.recv() if done: print(flush=True) print('download done', flush=True) # stop ticker when download process done self.ticker.cancel() break
def _model_process(model_function: Callable, conn: Connection, batch_size: int = -1, *, poll_period: float = 0.5): model: Callable[[Collection[str], Collection[Hashable]], Collection[str]] = model_function() if batch_size <= 0: batch_size = float('inf') while True: batch: List[Tuple[str, Hashable]] = [] while conn.poll() and len(batch) < batch_size: batch.append(conn.recv()) if not batch: time.sleep(poll_period) continue messages, dialog_ids = zip(*batch) responses = model(messages, dialog_ids) for response, dialog_id in zip(responses, dialog_ids): conn.send((response, dialog_id))
def _loop(self, task_reader: Connection, result_writer: Connection): if self.config.on_after_fork is not None: self.config.on_after_fork() executor = self._executor.run task_deserializer = AsyncTask.deserialize while True: if task_reader.poll(DURATION_MICROSECOND): message = task_reader.recv() task = task_deserializer(message) try: result = executor(task) if result is _SHUTDOWN_OBJECT: break except Exception as e: result = TaskError(e) result_writer.send((task.identifier, result)) if self.config.on_shutdown is not None: self.config.on_shutdown()
def recv_instruction_from_pipe(target_pipe: Connection): if target_pipe.poll(): instruction = target_pipe.recv() # Pass the exception to the main process if isinstance(instruction, ExceptionMessage): self._main_pipe.send((instruction, None)) # Invalid instruction object if not isinstance(instruction, GameInstruction): return GameInstruction(-1, comm.PlatformAction.NONE) # Invalid PlatformAction instruction if instruction.command != comm.PlatformAction.MOVE_LEFT and \ instruction.command != comm.PlatformAction.MOVE_RIGHT: instruction.command = comm.PlatformAction.NONE return instruction else: return GameInstruction(-1, comm.PlatformAction.NONE)
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters): ctrl_connection.send("failed to start rx mode") return False try: cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name]) except NotImplementedError: # Many SDRs like HackRF or AirSpy do not need to calculate SYNC_RX_CHUNK_SIZE # as default values are either fine or given by the hardware pass if cls.ASYNCHRONOUS: ret = cls.enter_async_receive_mode(data_connection, ctrl_connection) else: ret = cls.prepare_sync_receive(ctrl_connection) if ret != 0: ctrl_connection.send("failed to start rx mode") return False exit_requested = False ctrl_connection.send("successfully started rx mode") while not exit_requested: if cls.ASYNCHRONOUS: try: time.sleep(0.25) except KeyboardInterrupt: pass else: cls.receive_sync(data_connection) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False) if result == cls.Command.STOP.name: exit_requested = True break cls.shutdown_device(ctrl_connection, is_tx=False) data_connection.close() ctrl_connection.close()
def watcher_loop( signal_rx: PipeConnection, interval_seconds: float, first_run_immediate: bool, do_work: Callable[[Value], None], error_callback: Callable[[Exception], None], shared_state: Value, ): signal(SIGINT, MPWorker.empty_signal_handler) signal(SIGTERM, MPWorker.empty_signal_handler) configured_interval = interval_seconds if first_run_immediate: interval_seconds = 0 while not signal_rx.poll(timeout=interval_seconds): interval_seconds = configured_interval try: do_work(shared_state) except Exception as captured_exception: if error_callback is not None: error_callback(captured_exception) else: raise captured_exception
def _safe_queue_put(self, worker_id: int, item: Any, queue: mp.JoinableQueue, rx: Connection) -> bool: while True: # First we have to check to make sure the parent process is still alive # and consuming from the queue because there are circumstances where the # parent process can or exit stop consuming without automatically cleaning up # its children (the workers). # For example, when the parent process is killed with `kill -9`. # So the first thing we do is check to see if the parent has notified # us (the worker) to stop through the rx (receiver) connection. # Of course this only works if the parent was able to send out a notification, # which may not always be the case. So we have a backup check below. if rx.poll(): logger.warning( "worker %d received stop message from parent, exiting now", worker_id) queue.cancel_join_thread() return False # The is the backup check. # The file descriptor associated with the rx (receiver) connection will # be readable if and only if the parent process has exited. # NOTE (epwalsh): this doesn't work on Mac OS X with `start_method == "fork"` # for some reason, i.e. the file descriptor doesn't show as readable # after the parent process has died. fds, _, _ = select.select([rx.fileno()], [], [], 0) if fds: logger.warning( "worker %d parent process has died, exiting now", worker_id) queue.cancel_join_thread() return False # If we're down here the parent process is still alive to the best of our # knowledge, so we can continue putting things on the queue. try: queue.put(item, True, 0.1) return True except Full: continue
def poll_process( self, rx: Connection, analyzer_name: str, ) -> Iterator[ExecutionHit]: """ Keep polling the spawned Process, and yield any ExecutionHits. (This will probably disappear if Analyzers move to Docker images.) """ t = 0 while True: p_res = rx.poll(timeout=5) if not p_res: t += 1 LOGGER.info( f"Analyzer {analyzer_name} polled for for {t * 5} seconds without result" ) continue result: Optional[Any] = rx.recv() if isinstance(result, ExecutionComplete): self.logger.info(f"Analyzer {analyzer_name} execution complete") return # emit any hits to an S3 bucket if isinstance(result, ExecutionHit): self.logger.info( f"Analyzer {analyzer_name} emitting event for:" f"{result.analyzer_name} {result.root_node_key}" ) yield result assert not isinstance( result, ExecutionFailed ), f"Analyzer {analyzer_name} failed."
def state_farmer(q: Queue, exit_receiever: Connection, interval: int = 5): while not exit_receiever.poll(): q.put(get_sys_state()) time.sleep(interval)
def processor_process( connection: Connection, blueprint_path: str, init_params: dict, processor_dir: str, topic: str, source_topic: str, source_format: str, kafka_server: str, min_input_spacing: float, min_step_spacing: float, min_output_spacing: float, ): """ Runs the given blueprint as a processor Is meant to be run in a separate process :param connection: a connection object to communicate with the main process :param blueprint_path: the path to the blueprint folder :param init_params: the initialization parameters to the processor as a dictionary :param processor_dir: the directory the created process will run in :param topic: the topic the process will send results to :param source_topic: the topic the process will receive data from :param source_format: the byte format of the data the process will receive :param kafka_server: the address of the kafka bootstrap server the process will use :param min_input_spacing: the minimum time between each input to the processor :param min_step_spacing: the minimum time between each step function call on the processor :param min_output_spacing: the minimum time between each results retrieval from the processor :return: """ blueprint_path = os.path.realpath(blueprint_path) # Creates and sets, deleting if it already exists, the directory the process will use try: if os.path.exists(processor_dir): shutil.rmtree(processor_dir) os.makedirs(processor_dir) os.chdir(processor_dir) except PermissionError as e: connection.send({'type': 'error', 'value': e}) return start_time = -1 next_input_time = 0 next_step_time = 0 next_output_time = 0 # Imports the blueprints and initializes the processor from it try: processor_instance = SourceFileLoader( os.path.basename(blueprint_path), os.path.join(blueprint_path, '__init__.py')).load_module().P(**init_params) except TypeError as e: connection.send({'type': 'error', 'value': e}) return except ValueError as e: connection.send({'type': 'error', 'value': e}) return # Retrieves the inputs and outputs from the initialized processor if hasattr(processor_instance, 'outputs'): # Uses the outputs attribute from the processor if it exists outputs = [ Variable(v.valueReference, v.name) for v in processor_instance.outputs ] else: # Otherwise it has to create outputs from output_names outputs = [ Variable(i, name) for i, name in enumerate(processor_instance.output_names) ] if hasattr(processor_instance, 'inputs'): # Uses the inputs attribute from the processor if it exists inputs = [ Variable(v.valueReference, v.name) for v in processor_instance.inputs ] else: # Otherwise it has to create inputs from input_names inputs = [ Variable(i, name) for i, name in enumerate(processor_instance.input_names) ] # Add a helper attribute that lists outputs that are part of a matrix matrix_outputs = (processor_instance.matrix_outputs if hasattr( processor_instance, 'matrix_outputs') else []) initialized = True started = False byte_format = '<d' output_refs = [] input_refs = [] measurement_refs = [] measurement_proportions = [] topic_partition = TopicPartition(topic=source_topic, partition=0) output_buffer = bytearray() # Waits for the start signal from the main process start_params = None while start_params is None: msg = connection.recv() if msg['type'] == 'start': value = msg['value'] start_params = value['start_params'] output_refs = value['output_refs'] input_refs = value['input_refs'] measurement_refs = value['measurement_refs'] measurement_proportions = value['measurement_proportions'] byte_format = '<' + 'd' * (len(output_refs) + 1) elif msg['type'] == 'status': connection.send({ 'type': 'status', 'value': { 'outputs': outputs, 'inputs': inputs, # Add a helper attribute that lists outputs that are part of a matrix 'matrix_outputs': matrix_outputs, 'initialized': initialized, 'started': started, } }) elif msg['type'] == 'stop': try: processor_instance.stop() finally: return try: # Calls the processors start function if present if hasattr(processor_instance, 'start'): processor_instance.start(start_time=next_output_time, **start_params) except TypeError as e: connection.send({'type': 'error', 'value': e}) return except ValueError as e: connection.send({'type': 'error', 'value': e}) return # Use a custom time for results if the processor defines it if hasattr(processor_instance, "get_time"): processor_custom_time = True else: processor_custom_time = False try: # Initializes kafka consumer = kafka.KafkaConsumer(source_topic, bootstrap_servers=kafka_server) producer = kafka.KafkaProducer(value_serializer=bytes, bootstrap_servers=kafka_server) except: connection.send({'type': 'error', 'value': 'Kafka error'}) return started = True while True: try: while connection.poll(): # Handles new data from main process conn_msg = connection.recv() if conn_msg['type'] == 'outputs': output_refs = conn_msg['value'] byte_format = '<' + 'd' * (len(output_refs) + 1) elif conn_msg['type'] == 'inputs': input_refs, measurement_refs, measurement_proportions = conn_msg[ 'value'] elif conn_msg['type'] == 'stop': try: processor_instance.stop() finally: return elif conn_msg['type'] == 'status': connection.send({ 'type': 'status', 'value': { 'outputs': outputs, 'inputs': inputs, # Add a helper attribute that lists outputs that are part of a matrix 'matrix_outputs': matrix_outputs, 'initialized': initialized, 'started': started, } }) messages = consumer.poll(10) for msg in messages.get(topic_partition, []): # Handles new data from kafka for data in struct.iter_unpack(source_format, msg.value): current_time = data[0] if start_time < 0: start_time = current_time if next_input_time <= current_time or math.isclose( next_input_time, current_time, rel_tol=1e-15): # Retrieve measurements from data where ref is positive and use a constant where ref is negative measurements = [ (data[ref + 1] if ref >= 0 else 1) * measurement_proportions[i] for i, ref in enumerate(measurement_refs) ] processor_instance.set_inputs(input_refs, measurements) next_input_time = current_time + min_input_spacing if next_step_time <= current_time or math.isclose( next_step_time, current_time, rel_tol=1e-15): processor_instance.step(current_time - start_time) next_step_time = current_time + min_step_spacing if next_output_time <= current_time or math.isclose( next_output_time, current_time, rel_tol=1e-15): outputs = processor_instance.get_outputs(output_refs) if outputs is None: continue if processor_custom_time: current_time = processor_instance.get_time( ) + start_time output_buffer += struct.pack(byte_format, current_time, *outputs) if len(output_buffer) > 100 * len(output_refs): producer.send(topic=topic, value=output_buffer) output_buffer = bytearray() next_output_time = current_time + min_output_spacing except Exception as e: logger.exception(f'Exception in processor {processor_dir}') connection.send({'type': 'error', 'value': e}) return
def evolve(in_con:MConnection, out_con: MConnection, returnq: MQueue, settings: EnhanceModelSettings, optimal_primary_list: List[Gear], optimal_cost, cum_cost, secondaries=None, ev_set:EvolveSettings=None): if ev_set is None: ev_set = EvolveSettings() if secondaries is None: secondaries = settings[settings.P_FAIL_STACKER_SECONDARY] num_fs = ev_set.num_fs population_size = ev_set.pop_size ultra_elitism = ev_set.ultra_elite num_elites = ev_set.num_elites brood_size = ev_set.brood_size seent = set() this_seent = [] trait_dominance = ev_set.trait_dom mutation_rate = ev_set.mutation_rate extinction_epoch = ev_set.extinction_epoch max_mutation = ev_set.max_mutation f_fitness = fitness_funcs[ev_set.fitness_function] oppressive_mode = ev_set.oppressive_mode oppress_suprem = ev_set.penalize_supremacy best = cum_cost best_fsl = None lb = 0 def reg_prune(x: FailStackList): # sig = (x.starting_pos, x.secondary_gear, *x.secondary_map[:-1]) sig = (secondaries.index(x.secondary_gear), *x.get_gnome()) if sig in seent: return False else: seent.add(sig) this_seent.append(sig) # Send this signature to the other processes return True def check_pruned(x: FailStackList): sig = (secondaries.index(x.secondary_gear), *x.get_gnome()) return sig not in seent def accept_prune(x): return True check_2 = accept_prune check = accept_prune if oppress_suprem: check_2 = reg_prune check = check_pruned if oppressive_mode: check = reg_prune check_2 = reg_prune def get_randoms(size_): retlist = [ FailStackList(settings, choice(secondaries), optimal_primary_list, optimal_cost, cum_cost, num_fs=num_fs) for _ in range(0, size_)] [p.generate_secondary_map(randint(10, 60)) for p in retlist] return retlist def mutate(new_indiv): this_max_mutation = int(ceil(min(lb / 4, max_mutation))) for i, v in enumerate(new_indiv.secondary_map[:-1]): if random() < mutation_rate: new_v = v + randint(-this_max_mutation, this_max_mutation) new_indiv.secondary_map[i] = max(1, new_v) if random() < mutation_rate: new_s = new_indiv.starting_pos + randint(-this_max_mutation, this_max_mutation) new_indiv.starting_pos = min(max(5, new_s), 60) if random() < mutation_rate: new_indiv.secondary_gear = choice(secondaries) new_indiv.secondary_map[-1] = 300 best_fitness = fitness_func(optimal_cost, optimal_cost, cum_cost, cum_cost) this_brood_size = 0 population = get_randoms(population_size) epoch_mode = False while True: if lb > extinction_epoch: seent = set() population = get_randoms(population_size) lb = 0 [p.evaluate_map() for p in population] #pop_costs = best / numpy.array([f.fs_cum_cost for f in population]) # Bigger is better #fitness = numpy.sum(pop_costs, axis=1) pop_costs = numpy.array([f_fitness(f.fs_cost, optimal_cost, f.fs_cum_cost, cum_cost) for f in population]) # Bigger is better fitness = pop_costs sort_order = numpy.argsort(fitness, kind='mergesort') #for i in range(0, min(lb-15, brood_size)): # bad_fsl = population[sort_order[i]] # check_2(bad_fsl) if oppress_suprem: epoch_mode = lb > 5 if epoch_mode: for i in population[:this_brood_size]: check_2(i) brood_size = max(20, brood_size-lb) this_best_fitness = fitness[sort_order[-1]] if this_best_fitness > best_fitness: best_fitness = this_best_fitness best_fsl = population[sort_order[-1]] returnq.put((this_best_fitness, lb, (secondaries.index(best_fsl.secondary_gear), *best_fsl.get_gnome())), block=True) lb = 0 #best = numpy.min([best, best_fsl.fs_cum_cost], axis=0) new_pop = [] others_seent = [] try: while in_con.poll(): others_seent.extend(in_con.recv()) except EOFError: # Pipe broken: terminate loop out_con.close() return except BrokenPipeError: out_con.close() return for i in others_seent: this_len = len(seent) seent.add(tuple(i)) if len(seent) > this_len: this_seent.append(i) for i in range(0, brood_size): breeder1 = choice(sort_order[-num_elites:]) breeder1 = population[breeder1] if not epoch_mode and (best_fsl is not None) and (random() < ultra_elitism): breeder2 = best_fsl else: if random() > (lb * 0.1): breeder2 = choice(sort_order[-num_elites:]) breeder2 = population[breeder2] else: breeder2 = choice(sort_order) breeder2 = population[breeder2] offspring = FailStackList(settings, choice([breeder1.secondary_gear, breeder2.secondary_gear]), optimal_primary_list, optimal_cost, cum_cost, num_fs=num_fs) offspring.secondary_map = breeder1.secondary_map.copy() # this gets overwritten anyway for i in range(min(len(breeder1.secondary_map), len(breeder2.secondary_map))): if random() < trait_dominance: if random() < 0.5: offspring.secondary_map[i] = breeder1.secondary_map[i] else: offspring.secondary_map[i] = breeder2.secondary_map[i] else: offspring.secondary_map[i] = int(round((breeder1.secondary_map[i] + breeder2.secondary_map[i]) / 2.0)) if random() < trait_dominance: if random() < 0.5: offspring.starting_pos = breeder1.starting_pos else: offspring.starting_pos = breeder2.starting_pos else: offspring.starting_pos = int(round((breeder1.starting_pos + breeder2.starting_pos) / 2.0)) offspring.secondary_map[-1] = 300 mutate(offspring) if check(offspring): new_pop.append(offspring) this_brood_size = len(new_pop) new_pop.extend(get_randoms(population_size-len(new_pop))) population = new_pop if len(this_seent) > 0: out_con.send(this_seent) this_seent = [] lb += 1
def _slew_rate_process( self, axis: str, rate_target_pipe: Connection, axis_safe_event: Event, div_last_commanded_shared: Value, ): """Process for sending slew rate commands continuously until a target rate is achieved. This process helps the mount to accelerate smoothly, since this requires sending commands to the mount computer in rapid succession. Acceleration and slew rate step limits are enforced here. Commands are sent to the mount until the desired target slew rate is achieved, and then it will wait for a new rate target before sending further commands. Args: axis: The mount axis to be controlled by this process (one process per axis). rate_target_pipe: The receiving end connection to a multiprocessing pipe over which slew rate target values are sent. Rates are in degrees per second. axis_safe_event: When the axis is safed, meaning that motion is stopped, this event will be set. Otherwise, it will be cleared. div_last_commanded_shared: Shared memory storing the divisor value most recently commanded for this mount axis. """ # Ignore SIGINT in this process (will be handled in main process) signal.signal(signal.SIGINT, signal.SIG_IGN) # Last commanded rate is cached along with the time of last command to enforce acceleration # limit. Keep local copy of last commanded divisor to avoid accessing shared memory more # than necessary. axis_safe_event.set() div_target = 0 div_last_commanded = div_last_commanded_shared.value time_last_commanded = time.perf_counter() - 1e-3 shutdown = False while True: if shutdown == True: if div_last_commanded == 0: return # only try to receive from the pipe if a new rate target is waiting or if the last- # received rate target has been achieved, in which case we want to block elif rate_target_pipe.poll() or div_last_commanded == div_target: rate_target = rate_target_pipe.recv() # None is a special value indicating that it is time to shut down this process if rate_target is None: div_target = 0 shutdown = True else: div_target = self.slew_rate_to_div(rate_target) if div_target == div_last_commanded: continue time_current = time.perf_counter() # may not be able to achieve div_target if it exceeds rate accel or step limits rate_target = self.div_to_slew_rate(div_target) rate_last_commanded = self.div_to_slew_rate(div_last_commanded) rate_to_command = self._apply_rate_accel_limit( rate_target, time_current, rate_last_commanded, time_last_commanded) rate_to_command = self._apply_rate_step_limit( rate_to_command, rate_last_commanded) div = self.slew_rate_to_div(rate_to_command) # Clear this event before sending the actual commands since the state of the mount # is about to change and because if the commands fail for some reason the state of # the mount will be unknown and cannot be assumed to be safe. if div != 0: axis_safe_event.clear() try: self._set_divisor(axis, div, div_last_commanded) except Gemini2Exception as e: # dangerous to give up because this thread is critical for stopping mount motion # safely; better to keep trying to send commands to the bitter end print( f'Ignoring exception in {axis} slew rate command thread: {str(e)}' ) continue div_last_commanded_shared.value = div div_last_commanded = div time_last_commanded = time_current if div_last_commanded == 0: axis_safe_event.set()