def setup_device(cls, ctrl_connection: Connection, device_identifier): ret = usrp.open(device_identifier) ctrl_connection.send("OPEN:" + str(ret)) success = ret == 0 if success: device_repr = usrp.get_device_representation() ctrl_connection.send(device_repr) return success
def setup_device(cls, ctrl_connection: Connection, device_identifier): ret = hackrf.setup(device_identifier) msg = "SETUP" if device_identifier: msg += " ({})".format(device_identifier) msg += ": "+str(ret) ctrl_connection.send(msg) return ret == 0
def shutdown_device(cls, ctrl_conn: Connection): logger.debug("HackRF: closing device") ret = hackrf.close() ctrl_conn.send("CLOSE:" + str(ret)) ret = hackrf.exit() ctrl_conn.send("EXIT:" + str(ret)) return True
def prepare_sync_send(cls, ctrl_connection: Connection): try: cls.pyaudio_stream = cls.pyaudio_handle.open(format=pyaudio.paFloat32, channels=2, rate=cls.SAMPLE_RATE, output=True) ctrl_connection.send("Successfully started pyaudio stream") return 0 except Exception as e: logger.exception(e) ctrl_connection.send("Failed to start pyaudio stream")
def setup_device(cls, ctrl_connection: Connection, device_identifier): ret = limesdr.open() ctrl_connection.send("OPEN:" + str(ret)) limesdr.disable_all_channels() if ret != 0: return False ret = limesdr.init() ctrl_connection.send("INIT:" + str(ret)) return ret == 0
def prepare_sync_receive(cls, ctrl_connection: Connection): try: cls.pyaudio_stream = cls.pyaudio_handle.open(format=pyaudio.paFloat32, channels=2, rate=cls.SAMPLE_RATE, input=True, frames_per_buffer=cls.CHUNK_SIZE) ctrl_connection.send("Successfully started pyaudio stream") return 0 except Exception as e: logger.exception(e) ctrl_connection.send("Failed to start pyaudio stream")
def generate_data(connection: Connection, num_samples=32768): frequency = 0.1 divisor = 200 pos = 0 while True: result = np.zeros(num_samples, dtype=np.complex64) result.real = np.cos(2 * np.pi * frequency * np.arange(pos, pos + num_samples)) result.imag = np.sin(2 * np.pi * frequency * np.arange(pos, pos + num_samples)) pos += num_samples if pos / num_samples >= divisor: frequency *= 2 if frequency >= 1: frequency = 0.1 pos = 0 connection.send(result)
def setup_device(cls, ctrl_connection: Connection, device_identifier): device_identifier = device_identifier if isinstance(device_identifier, str) else "" try: device_identifier = re.search("\[.*\]", device_identifier).groups(0) except (IndexError, AttributeError): pass if not device_identifier: _, uris = plutosdr.scan_devices() try: device_identifier = uris[0] except IndexError: ctrl_connection.send("Could find a connected PlutoSDR") return False ret = plutosdr.open(device_identifier) ctrl_connection.send("OPEN ({}):{}".format(device_identifier, ret)) return ret == 0
def shutdown_device(cls, ctrl_conn: Connection, is_tx: bool): if is_tx: result = hackrf.stop_tx_mode() ctrl_conn.send("STOP TX MODE:" + str(result)) else: result = hackrf.stop_rx_mode() ctrl_conn.send("STOP RX MODE:" + str(result)) result = hackrf.close() ctrl_conn.send("CLOSE:" + str(result)) result = hackrf.exit() ctrl_conn.send("EXIT:" + str(result)) return True
def setup_device(cls, ctrl_connection: Connection, device_identifier): ctrl_connection.send("Initializing pyaudio...") try: cls.pyaudio_handle = pyaudio.PyAudio() ctrl_connection.send("Initialized pyaudio") return True except Exception as e: logger.exception(e) ctrl_connection.send("Failed to initialize pyaudio")
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters): ctrl_connection.send("failed to start tx mode") return False if cls.ASYNCHRONOUS: ret = cls.enter_async_send_mode(send_config.get_data_to_send) else: ret = cls.prepare_sync_send(ctrl_connection) if ret != 0: ctrl_connection.send("failed to start tx mode") return False exit_requested = False buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE if not cls.ASYNCHRONOUS and buffer_size == 0: logger.warning("Send buffer size is zero!") ctrl_connection.send("successfully started tx mode") while not exit_requested and not send_config.sending_is_finished(): if cls.ASYNCHRONOUS: try: time.sleep(0.5) except KeyboardInterrupt: pass else: cls.send_sync(send_config.get_data_to_send(buffer_size)) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True) if result == cls.Command.STOP.name: exit_requested = True break if not cls.ASYNCHRONOUS: # Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure # that the send buffer on the SDR is cleared time.sleep(0.75) if exit_requested: logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__)) if send_config.sending_is_finished(): logger.debug("{}: sending is finished.".format(cls.__class__.__name__)) cls.shutdown_device(ctrl_connection, is_tx=True) ctrl_connection.close()
def init_device(cls, ctrl_connection: Connection, is_tx: bool, parameters: OrderedDict): if not cls.setup_device(ctrl_connection, device_identifier=parameters["identifier"]): return False limesdr.enable_channel(True, is_tx, parameters[cls.Command.SET_CHANNEL_INDEX.name]) limesdr.set_tx(is_tx) for parameter, value in parameters.items(): cls.process_command((parameter, value), ctrl_connection, is_tx) antennas = limesdr.get_antenna_list() ctrl_connection.send("Current normalized gain is {0:.2f}".format(limesdr.get_normalized_gain())) ctrl_connection.send("Current antenna is {0}".format(antennas[limesdr.get_antenna()])) ctrl_connection.send("Current chip temperature is {0:.2f}°C".format(limesdr.get_chip_temperature())) return True
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict): if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters): ctrl_connection.send("failed to start rx mode") return False try: cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name]) except NotImplementedError: # Many SDRs like HackRF or AirSpy do not need to calculate SYNC_RX_CHUNK_SIZE # as default values are either fine or given by the hardware pass if cls.ASYNCHRONOUS: ret = cls.enter_async_receive_mode(data_connection, ctrl_connection) else: ret = cls.prepare_sync_receive(ctrl_connection) if ret != 0: ctrl_connection.send("failed to start rx mode") return False exit_requested = False ctrl_connection.send("successfully started rx mode") while not exit_requested: if cls.ASYNCHRONOUS: try: time.sleep(0.25) except KeyboardInterrupt: pass else: cls.receive_sync(data_connection) while ctrl_connection.poll(): result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False) if result == cls.Command.STOP.name: exit_requested = True break cls.shutdown_device(ctrl_connection, is_tx=False) data_connection.close() ctrl_connection.close()
def prepare_sync_receive(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing stream...") usrp.setup_stream() usrp.start_stream(cls.READ_SAMPLES) ctrl_connection.send("Initialized stream")
def _custom_worker_shared_memory(index: int, env_fn: Callable[[], Env], pipe: Connection, parent_pipe: Connection, shared_memory, error_queue: Queue, in_series: int = None): """Copied and modified from `gym.vector.async_vector_env`. Args: index ([type]): [description] env_fn ([type]): [description] pipe ([type]): [description] parent_pipe ([type]): [description] shared_memory ([type]): [description] error_queue ([type]): [description] Raises: RuntimeError: [description] """ process_name = mp.current_process().name # print(f"Current process name: {process_name}") assert shared_memory is not None env = env_fn() observation_space = env.observation_space parent_pipe.close() def step_fn(actions): observation, reward, done, info = env.step(actions) if isinstance(env, VectorEnv): # Do nothing: Since the env is a VectorEnv, it will automatically # reset the env if needed in 'step' and return the initial # observation instead of the final observation. return observation, reward, done, info if done: if info is None: info = {} assert isinstance(info, dict) info[FINAL_STATE_KEY] = observation observation = env.reset() return observation, reward, done, info try: while True: command, data = pipe.recv() # print(f"Worker {index} received command {command}") if command == Commands.reset: observation = env.reset() write_to_shared_memory(index, observation, shared_memory, observation_space) pipe.send((None, True)) elif command == Commands.step: observation, reward, done, info = step_fn(data) write_to_shared_memory(index, observation, shared_memory, observation_space) pipe.send(((None, reward, done, info), True)) elif command == Commands.seed: env.seed(data) pipe.send((None, True)) elif command == Commands.close: pipe.send((None, True)) break elif command == '_check_observation_space': pipe.send((data == observation_space, True)) # Below this: added commands. elif command == Commands.apply: assert callable(data) function = data results = function(env) pipe.send((results, True)) elif command == Commands.render: pipe.send(env.render(mode="rgb_array")) else: raise RuntimeError('Received unknown command `{0}`. Must ' 'be one of {`reset`, `step`, `seed`, `close`, ' '`_check_observation_space`, `apply`}.'.format(command)) except (KeyboardInterrupt, Exception): error_queue.put((index,) + sys.exc_info()[:2]) pipe.send((None, False)) finally: env.close()
def enter_async_receive_mode(cls, data_connection: Connection, ctrl_connection: Connection): ret = airspy.start_rx(data_connection.send_bytes) ctrl_connection.send("Start RX MODE:" + str(ret))
def set_state_value(conn: Connection, key, value): state_message = ModuleMessage("PRGH", "set_state_data", (key, value)) conn.send(state_message)
def prepare_sync_receive(cls, ctrl_connection: Connection): ret = rtlsdr.reset_buffer() ctrl_connection.send("RESET_BUFFER:" + str(ret)) return ret == 0
def enter_async_receive_mode(cls, data_connection: Connection, ctrl_connection: Connection): ret = hackrf.start_rx_mode(data_connection.send_bytes) ctrl_connection.send("Start RX MODE:" + str(ret)) return ret
def _run_file_processor( result_channel: MultiprocessingConnection, parent_channel: MultiprocessingConnection, file_path: str, pickle_dags: bool, dag_ids: Optional[List[str]], thread_name: str, callback_requests: List[CallbackRequest], ) -> None: """ Process the given file. :param result_channel: the connection to use for passing back the result :type result_channel: multiprocessing.Connection :param parent_channel: the parent end of the channel to close in the child :type parent_channel: multiprocessing.Connection :param file_path: the file to process :type file_path: str :param pickle_dags: whether to pickle the DAGs found in the file and save them to the DB :type pickle_dags: bool :param dag_ids: if specified, only examine DAG ID's that are in this list :type dag_ids: list[str] :param thread_name: the name to use for the process that is launched :type thread_name: str :param callback_requests: failure callback to execute :type callback_requests: List[airflow.utils.callback_requests.CallbackRequest] :return: the process that was launched :rtype: multiprocessing.Process """ # This helper runs in the newly created process log: logging.Logger = logging.getLogger("airflow.processor") # Since we share all open FDs from the parent, we need to close the parent side of the pipe here in # the child, else it won't get closed properly until we exit. log.info("Closing parent pipe") parent_channel.close() del parent_channel set_context(log, file_path) setproctitle(f"airflow scheduler - DagFileProcessor {file_path}") try: # redirect stdout/stderr to log with redirect_stdout(StreamLogWriter( log, logging.INFO)), redirect_stderr( StreamLogWriter(log, logging.WARN)), Stats.timer() as timer: # Re-configure the ORM engine as there are issues with multiple processes settings.configure_orm() # Change the thread name to differentiate log lines. This is # really a separate process, but changing the name of the # process doesn't work, so changing the thread name instead. threading.current_thread().name = thread_name log.info("Started process (PID=%s) to work on %s", os.getpid(), file_path) dag_file_processor = DagFileProcessor(dag_ids=dag_ids, log=log) result: Tuple[int, int] = dag_file_processor.process_file( file_path=file_path, pickle_dags=pickle_dags, callback_requests=callback_requests, ) result_channel.send(result) log.info("Processing %s took %.3f seconds", file_path, timer.duration) except Exception: # Log exceptions through the logging framework. log.exception("Got an exception! Propagating...") raise finally: # We re-initialized the ORM within this Process above so we need to # tear it down manually here settings.dispose_orm() result_channel.close()
def _worker( parent: connection.Connection, p: connection.Connection, env_fn_wrapper: CloudpickleWrapper, obs_bufs: Optional[Union[dict, tuple, ShArray]] = None, ) -> None: def _encode_obs(obs: Union[dict, tuple, np.ndarray], buffer: Union[dict, tuple, ShArray]) -> None: if isinstance(obs, np.ndarray) and isinstance(buffer, ShArray): buffer.save(obs) elif isinstance(obs, tuple) and isinstance(buffer, tuple): for o, b in zip(obs, buffer): _encode_obs(o, b) elif isinstance(obs, dict) and isinstance(buffer, dict): for k in obs.keys(): _encode_obs(obs[k], buffer[k]) return None parent.close() env = env_fn_wrapper.data() try: while True: try: cmd, data = p.recv() except EOFError: # the pipe has been closed p.close() break if cmd == "step": obs, reward, done, info = env.step(data) if obs_bufs is not None: _encode_obs(obs, obs_bufs) obs = None p.send((obs, reward, done, info)) elif cmd == "reset": obs = env.reset() if obs_bufs is not None: _encode_obs(obs, obs_bufs) obs = None p.send(obs) elif cmd == "close": p.send(env.close()) p.close() break elif cmd == "render": p.send(env.render(**data) if hasattr(env, "render") else None) elif cmd == "seed": p.send(env.seed(data) if hasattr(env, "seed") else None) elif cmd == "getattr": p.send(getattr(env, data) if hasattr(env, data) else None) else: p.close() raise NotImplementedError except KeyboardInterrupt: p.close()
def prepare_sync_send(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing stream...") limesdr.setup_stream(cls.SEND_FIFO_SIZE) ret = limesdr.start_stream() ctrl_connection.send("Initialize stream:{0}".format(ret)) return ret
def prepare_sync_send(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing stream...") usrp.setup_stream() ret = usrp.start_stream(0) ctrl_connection.send("Initialize stream:{0}".format(ret))
def prepare_sync_send(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing PlutoSDR...") ret = plutosdr.setup_tx(cls.SYNC_TX_CHUNK_SIZE // 2) return ret
def prepare_sync_send(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing BladeRF...") ret = bladerf.prepare_sync() return ret
def setup_device(cls, ctrl_connection: Connection, device_identifier): ret = hackrf.setup() ctrl_connection.send("SETUP:" + str(ret)) return ret == 0
def run(node_config: List[int], num_events: int, duration: int, windows: str, agg_functions: str, is_single_node: bool = False, is_fixed_events: bool = False, process_log_dir_pipe: Connection = None): assert_valid_node_config(node_config) num_nodes = sum(node_config) + 1 log_dir = get_log_dir(num_nodes, num_events, duration) log_dir = create_log_dir(log_dir) if process_log_dir_pipe is not None: sys.stdout = open(os.path.join(log_dir, "main.log"), "w") print(f"Writing logs to {log_dir}\n") all_hosts = get_hosts_for_run(node_config) all_hosts.insert(0, [ROOT_HOST]) flat_hosts = [host for level in all_hosts for host in level] print(f"All hosts ({len(flat_hosts)}): {' '.join(flat_hosts)}") port_iter = iter(ALL_PORTS) print("Uploading benchmark arguments on all nodes...") # Upload for root node named_hosts = [(ROOT_HOST, "root")] upload_root_params(len(all_hosts[1]), windows, agg_functions, is_single_node) # Upload for intermediate nodes for level_id, level_hosts in enumerate(all_hosts[1:-2], 1): parent_nodes = all_hosts[level_id - 1] num_parents = len(parent_nodes) num_children = len(all_hosts[level_id + 1]) assert num_children % len(level_hosts) == 0 num_children_per_node = num_children // len(level_hosts) for node_id, level_host in enumerate(level_hosts): parent_host = parent_nodes[node_id % num_parents] node_level_id = (10 * level_id) + node_id upload_intermediate_params(level_host, node_level_id, num_children_per_node, parent_host, is_single_node) named_hosts.append((level_host, f"merger-{node_level_id}")) parent_nodes = all_hosts[-3] child_hosts = all_hosts[-2] stream_hosts = all_hosts[-1] num_parents = len(parent_nodes) num_children = len(child_hosts) num_streams = len(stream_hosts) # # Upload for child nodes # num_streams_per_child = num_streams // num_children # for child_id, child_host in enumerate(child_hosts): # parent_host = parent_nodes[child_id % num_parents] # upload_child_params(child_host, parent_host, child_id, num_streams_per_child, is_single_node) # named_hosts.append((child_host, f"child-{child_id}")) # Upload for child and stream nodes num_streams_per_child = num_streams // num_children for child_id, child_host in enumerate(child_hosts): parent_host = parent_nodes[child_id % num_parents] child_port = next(port_iter) upload_child_params(child_host, parent_host, child_port, child_id, num_streams_per_child, is_single_node) named_hosts.append((child_host, f"child-{child_id}")) upload_stream_params(child_host, child_id, child_port, num_events, duration, agg_functions, is_fixed_events) named_hosts.append((child_host, f"stream-{child_id}")) # TODO # # Upload for stream nodes # for stream_id, stream_host in enumerate(stream_hosts): # parent_host = child_hosts[stream_id % num_children] # upload_stream_params(stream_host, stream_id, parent_host, num_events, duration, # agg_functions, is_fixed_events) # named_hosts.append((stream_host, f"stream-{stream_id}")) print("Starting `run.sh` on all nodes.\n") max_run_duration = duration + 30 threads = [] for host, name in named_hosts: mode = "" if name.startswith("child"): mode = "CHILD" if name.startswith("stream"): mode = "STREAM" thread = Thread(target=run_host, args=(host, name, log_dir, max_run_duration, mode), name=f"thread-{name}") thread.start() threads.append(thread) # Give nodes time to start running time.sleep(5) check_complete(max_run_duration, flat_hosts, complete_check_fn) kill_command = "pkill -9 -f /home/hadoop/benson/openjdk12/bin/java" print("Ending script by killing all PIDs...") for host in flat_hosts: ssh_command(host, kill_command) for thread in threads: thread.join() print("Joined all run threads.") if process_log_dir_pipe is not None: process_log_dir_pipe.send(log_dir) return log_dir
def env_worker_multiple_envs(conn: Connection, **env_kwargs): envs = {} def get_env(field_id): if field_id in envs: return envs[field_id] else: gameplay = Ccreate_start_conditions() env = Haxball(gameplay=gameplay, **env_kwargs) envs[field_id] = env return env while True: cmd, data = conn.recv() if cmd == 'step': obss = [] rews = [] dones = [] infos = [] field_ids, a1s, a2s = zip(*data) players_i = [] for i, field_id in enumerate(field_ids): field_id = field_ids[i] a1 = a1s[i] a2 = a2s[i] # rf1 = rf1s[i] # rf2 = rf2s[i] players_i.append(field_id * 2) players_i.append(field_id * 2 + 1) env = get_env(field_id) # Le azioni non sono immediate # steps_to_wait = max(1, np.random.poisson(lam=1)) env.step_physics(1) env.step_async(a1, red_team=True) env.step_async(a2, red_team=False) # Le azioni sfalsate delay = int(np.round(np.clip(np.random.normal(8, 1), 5, 11))) if random.random() < 0.5: pause1 = delay // 2 pause2 = delay - pause1 else: pause2 = delay // 2 pause1 = delay - pause2 # env.step_physics(pause1) env.step_physics(pause1 + pause2) # env.step_async(a2, red_team=False) # env.step_physics(1) # Le misure di obs non sono immediate # steps_to_wait = max(1, np.random.poisson(lam=2)) # steps_to_wait = 1 # steps_to_wait = max(1, np.random.poisson(lam=1)) # env.step_physics(steps_to_wait) is_done = False for red_team in (True, False): obs, rew, done, info = env.step_wait(red_team=red_team) obss.append(obs) rews.append(rew) dones.append(done) info['field_id'] = field_id info[ 'players_i'] = field_id * 2 if red_team else field_id * 2 + 1 infos.append(info) is_done |= done # if red_team and not done: # env.step_physics(pause2) if is_done: env.reset() res = field_ids, players_i, np.array(obss), np.array( rews), np.array(dones), np.array(infos) conn.send(res) elif cmd == 'reset': field_id = data env = get_env(field_id) ob = env.reset() conn.send([ob, ob]) elif cmd == 'render': field_id = data env = get_env(field_id) res = env.render(mode='rgb_array') conn.send(res) elif cmd == 'close': conn.close() break elif cmd == 'get_spaces_spec': field_id = data env = get_env(field_id) conn.send((env.observation_space, env.action_space, env.spec)) else: raise NotImplementedError
def prepare_sync_receive(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing PlutoSDR..") ret = plutosdr.setup_rx(cls.SYNC_RX_CHUNK_SIZE) return ret
def wrapped_run(pipe: Connection): result = job.run() pipe.send(result)
def mailer_process( *, queue: mp.Queue, error_channel: mpc.Connection, quit_event: mp.Event, rate_limiter: RateLimiter = None, rate_meter: RateMeter = None, ): """ Main function of the processes responsible for sending email messages. This process pulls `(message: EmailMessage, event: CampaignSentEvent)` tuples from the work queue, tries to send the message and saves the result on the `event`. Whenever an unexpected error happens (i.e. which does not seem to be linked to a particular recipient), the process signals the error on the `error_channel` by sending the campaign id. It monitors its own connection to the mail service, and resets it every `nuntius.app_settings.MAX_MESSAGES_PER_CONNECTION` messages. :param queue: The work queue on which tuples (EmailMessage, CampaignSentEvent) are received. :type queue: class:`multiprocessing.Queue` :param error_channel: channel on which campaign ids of failing campaigns are sent. :type error_channel: class:`multiprocessing.connection.Connection` :param quit_event: event that may be used by the main process to signal to senders they need to quit :type quit_event: class:`multiprocessing.Event` :param rate_limiter: rate limiter to limit the rate of message sendings. :type rate_limiter: class:`nuntius.utils.processes.RateLimiter` :param rate_meter: rate meter to allow measuring the sending speed :type rate_meter: class:`nuntius.utils.processes.RateMeter` """ message: EmailMessage sent_event_id: int try: with ConnectionManager(quit_event) as connection_manager: while True: # the timeout allows the loop to start again every few seconds so that the # quit_event is checked and the process can quit if it has to. message, sent_event_id = get_from_queue_or_quit( queue, event=quit_event, polling_period=app_settings.POLLING_INTERVAL, ) sent_event_qs = CampaignSentEvent.objects.filter(id=sent_event_id) # messages generated by the campaign manager should only have one recipient email = message.to[0] # rate limit just before sending if rate_limiter: rate_limiter.take() try: connection_manager.send_message(message) except (smtplib.SMTPRecipientsRefused, AnymailRecipientsRefused): # exceptions linked to a specific recipient need not stop the sending CampaignSentEvent.objects.filter(id=sent_event_id).update( result=CampaignSentStatusType.BLOCKED ) except GracefulExit: raise except Exception: campaign = Campaign.objects.get(campaignsentevent__id=sent_event_id) error_channel.send(campaign.id) logger.error( _("Error while sending email for campaign %(campaign)s") % repr(campaign), exc_info=True, ) else: if rate_meter: rate_meter.count_up() if hasattr(message, "anymail_status"): if message.anymail_status.recipients[email].status in [ "invalid", "rejected", "failed", ]: sent_event_qs.update(result=CampaignSentStatusType.REJECTED) else: sent_event_qs.update( result=CampaignSentStatusType.UNKNOWN, esp_message_id=message.anymail_status.recipients[ email ].message_id, ) else: sent_event_qs.update(result=CampaignSentStatusType.UNKNOWN) except GracefulExit: return
def setup_device(cls, ctrl_connection: Connection, device_identifier): ret = airspy.open() ctrl_connection.send("OPEN:" + str(ret)) return ret == 0
def pusher_process( *, queue: mp.Queue, error_channel: mpc.Connection, quit_event: mp.Event, rate_limiter: RateLimiter = None, rate_meter: RateMeter = None, ): """ Main function of the processes responsible for sending push notifications. This process pulls `(push_notification, event: PushCampaignSentEvent)` tuples from the work queue, tries to send the push and saves the result on the `event`. Whenever an unexpected error happens (i.e. which does not seem to be linked to a particular recipient), the process signals the error on the `error_channel` by sending the campaign id. It monitors its own connection to the mail service, and resets it every `nuntius.app_settings.MAX_MESSAGES_PER_CONNECTION` messages. :param queue: The work queue on which tuples (push_notification, CampaignSentEvent) are received. :type queue: class:`multiprocessing.Queue` :param error_channel: channel on which campaign ids of failing campaigns are sent. :type error_channel: class:`multiprocessing.connection.Connection` :param quit_event: event that may be used by the main process to signal to senders they need to quit :type quit_event: class:`multiprocessing.Event` :param rate_limiter: rate limiter to limit the rate of message sendings. :type rate_limiter: class:`nuntius.utils.processes.RateLimiter` :param rate_meter: rate meter to allow measuring the sending speed :type rate_meter: class:`nuntius.utils.processes.RateMeter` """ notification: dict push_sent_event_id: int try: with PushManager(quit_event) as push_manager: while True: # the timeout allows the loop to start again every few seconds so that the # quit_event is checked and the process can quit if it has to. notification, push_sent_event_id = get_from_queue_or_quit( queue, event=quit_event, polling_period=app_settings.POLLING_INTERVAL, ) # rate limit just before sending if rate_limiter: rate_limiter.take() try: push_sent_event = PushCampaignSentEvent.objects.get( id=push_sent_event_id ) push_manager.push(notification, push_sent_event) except GracefulExit: raise except PushCampaignSentEvent.DoesNotExist: logger.error( _( f"Push campaign sent event with id '{push_sent_event_id}' not found" ), exc_info=True, ) except Exception: push_campaign = PushCampaign.objects.get( pushcampaignsentevent__id=push_sent_event_id ) error_channel.send(push_campaign.id) logger.error( _( f"Error while pushing notification for campaign {repr(push_campaign)}" ), exc_info=True, ) else: if rate_meter: rate_meter.count_up() except GracefulExit: return
def setup_device(cls, ctrl_connection: Connection, device_identifier): # identifier gets set in self.receive_process_arguments device_number = int(device_identifier) ret = rtlsdr.open(device_number) ctrl_connection.send("OPEN:" + str(ret)) return ret == 0
def ret_through_conn(*args, fn: Callable, send_conn: Connection, **kwargs) -> None: send_conn.send(fn(*args, **kwargs))
def evolve(in_con:MConnection, out_con: MConnection, returnq: MQueue, settings: EnhanceModelSettings, optimal_primary_list: List[Gear], optimal_cost, cum_cost, secondaries=None, ev_set:EvolveSettings=None): if ev_set is None: ev_set = EvolveSettings() if secondaries is None: secondaries = settings[settings.P_FAIL_STACKER_SECONDARY] num_fs = ev_set.num_fs population_size = ev_set.pop_size ultra_elitism = ev_set.ultra_elite num_elites = ev_set.num_elites brood_size = ev_set.brood_size seent = set() this_seent = [] trait_dominance = ev_set.trait_dom mutation_rate = ev_set.mutation_rate extinction_epoch = ev_set.extinction_epoch max_mutation = ev_set.max_mutation f_fitness = fitness_funcs[ev_set.fitness_function] oppressive_mode = ev_set.oppressive_mode oppress_suprem = ev_set.penalize_supremacy best = cum_cost best_fsl = None lb = 0 def reg_prune(x: FailStackList): # sig = (x.starting_pos, x.secondary_gear, *x.secondary_map[:-1]) sig = (secondaries.index(x.secondary_gear), *x.get_gnome()) if sig in seent: return False else: seent.add(sig) this_seent.append(sig) # Send this signature to the other processes return True def check_pruned(x: FailStackList): sig = (secondaries.index(x.secondary_gear), *x.get_gnome()) return sig not in seent def accept_prune(x): return True check_2 = accept_prune check = accept_prune if oppress_suprem: check_2 = reg_prune check = check_pruned if oppressive_mode: check = reg_prune check_2 = reg_prune def get_randoms(size_): retlist = [ FailStackList(settings, choice(secondaries), optimal_primary_list, optimal_cost, cum_cost, num_fs=num_fs) for _ in range(0, size_)] [p.generate_secondary_map(randint(10, 60)) for p in retlist] return retlist def mutate(new_indiv): this_max_mutation = int(ceil(min(lb / 4, max_mutation))) for i, v in enumerate(new_indiv.secondary_map[:-1]): if random() < mutation_rate: new_v = v + randint(-this_max_mutation, this_max_mutation) new_indiv.secondary_map[i] = max(1, new_v) if random() < mutation_rate: new_s = new_indiv.starting_pos + randint(-this_max_mutation, this_max_mutation) new_indiv.starting_pos = min(max(5, new_s), 60) if random() < mutation_rate: new_indiv.secondary_gear = choice(secondaries) new_indiv.secondary_map[-1] = 300 best_fitness = fitness_func(optimal_cost, optimal_cost, cum_cost, cum_cost) this_brood_size = 0 population = get_randoms(population_size) epoch_mode = False while True: if lb > extinction_epoch: seent = set() population = get_randoms(population_size) lb = 0 [p.evaluate_map() for p in population] #pop_costs = best / numpy.array([f.fs_cum_cost for f in population]) # Bigger is better #fitness = numpy.sum(pop_costs, axis=1) pop_costs = numpy.array([f_fitness(f.fs_cost, optimal_cost, f.fs_cum_cost, cum_cost) for f in population]) # Bigger is better fitness = pop_costs sort_order = numpy.argsort(fitness, kind='mergesort') #for i in range(0, min(lb-15, brood_size)): # bad_fsl = population[sort_order[i]] # check_2(bad_fsl) if oppress_suprem: epoch_mode = lb > 5 if epoch_mode: for i in population[:this_brood_size]: check_2(i) brood_size = max(20, brood_size-lb) this_best_fitness = fitness[sort_order[-1]] if this_best_fitness > best_fitness: best_fitness = this_best_fitness best_fsl = population[sort_order[-1]] returnq.put((this_best_fitness, lb, (secondaries.index(best_fsl.secondary_gear), *best_fsl.get_gnome())), block=True) lb = 0 #best = numpy.min([best, best_fsl.fs_cum_cost], axis=0) new_pop = [] others_seent = [] try: while in_con.poll(): others_seent.extend(in_con.recv()) except EOFError: # Pipe broken: terminate loop out_con.close() return except BrokenPipeError: out_con.close() return for i in others_seent: this_len = len(seent) seent.add(tuple(i)) if len(seent) > this_len: this_seent.append(i) for i in range(0, brood_size): breeder1 = choice(sort_order[-num_elites:]) breeder1 = population[breeder1] if not epoch_mode and (best_fsl is not None) and (random() < ultra_elitism): breeder2 = best_fsl else: if random() > (lb * 0.1): breeder2 = choice(sort_order[-num_elites:]) breeder2 = population[breeder2] else: breeder2 = choice(sort_order) breeder2 = population[breeder2] offspring = FailStackList(settings, choice([breeder1.secondary_gear, breeder2.secondary_gear]), optimal_primary_list, optimal_cost, cum_cost, num_fs=num_fs) offspring.secondary_map = breeder1.secondary_map.copy() # this gets overwritten anyway for i in range(min(len(breeder1.secondary_map), len(breeder2.secondary_map))): if random() < trait_dominance: if random() < 0.5: offspring.secondary_map[i] = breeder1.secondary_map[i] else: offspring.secondary_map[i] = breeder2.secondary_map[i] else: offspring.secondary_map[i] = int(round((breeder1.secondary_map[i] + breeder2.secondary_map[i]) / 2.0)) if random() < trait_dominance: if random() < 0.5: offspring.starting_pos = breeder1.starting_pos else: offspring.starting_pos = breeder2.starting_pos else: offspring.starting_pos = int(round((breeder1.starting_pos + breeder2.starting_pos) / 2.0)) offspring.secondary_map[-1] = 300 mutate(offspring) if check(offspring): new_pop.append(offspring) this_brood_size = len(new_pop) new_pop.extend(get_randoms(population_size-len(new_pop))) population = new_pop if len(this_seent) > 0: out_con.send(this_seent) this_seent = [] lb += 1
def ConnSend(connection:Connection, msg): connection.send(msg)
def prepare_sync_receive(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing stream...") usrp.setup_stream() return usrp.start_stream(cls.SYNC_RX_CHUNK_SIZE)
def setup_device(cls, ctrl_connection: Connection, device_identifier): # identifier gets set in self.receive_process_arguments device_number = int(device_identifier) ret = rtlsdr.open(device_number) ctrl_connection.send("OPEN (#{}):{}".format(device_number, ret)) return ret == 0
def terminal_process(termtype: str, w: int, h: int, pw: int, ph: int, subproc_pipe: Connection): """ Avoid Python curses singleton bug by stuffing Terminal in a subprocess and proxying calls/responses via Pipe """ subproc_term = SubprocessTerminal(termtype, w, h, pw, ph) while True: try: given_attr, args, kwargs = subproc_pipe.recv() except KeyboardInterrupt: return if debug_term: log.debug(f'proxy received: {given_attr}, {args!r}, ' f'{kwargs!r}') # exit sentinel if given_attr is None: if debug_term: log.debug(f'term={subproc_term}/pid={os.getpid()} exit') break # special attribute -- a context manager, enter it naturally, exit # unnaturally (even, prematurely), with the exit value ready for # our client side, this is only possible because blessed doesn't # use any state or time-sensitive values, only terminal sequences, # and these CM's are the only ones without side-effects. if given_attr.startswith('!CTX'): # here, we feel the real punishment of side-effects... sideeffect_stream = subproc_term.stream.getvalue() assert not sideeffect_stream, ('should be empty', sideeffect_stream) given_attr = given_attr[len('!CTX'):] if debug_term: log.debug(f'context attr: {given_attr}') with getattr(subproc_term, given_attr)(*args, **kwargs) \ as enter_result: enter_side_effect = subproc_term.stream.getvalue() subproc_term.stream.truncate(0) subproc_term.stream.seek(0) if debug_term: log.debug('enter_result, enter_side_effect = ' f'{enter_result!r}, {enter_side_effect!r}') subproc_pipe.send((enter_result, enter_side_effect)) exit_side_effect = subproc_term.stream.getvalue() subproc_term.stream.truncate(0) subproc_term.stream.seek(0) subproc_pipe.send(exit_side_effect) elif given_attr.startswith('!CALL'): given_attr = given_attr[len('!CALL'):] matching_attr = getattr(subproc_term, given_attr) if debug_term: log.debug(f'callable attr: {given_attr}') subproc_pipe.send(matching_attr(*args, **kwargs)) else: if debug_term: log.debug(f'attr: {given_attr}') assert len(args) == len(kwargs) == 0, (args, kwargs) matching_attr = getattr(subproc_term, given_attr) if debug_term: log.debug(f'value: {matching_attr!r}') subproc_pipe.send(matching_attr)
def game_loop(self, fps: int, level: int, \ instruct_pipe: Connection, scene_info_pipe: Connection, main_pipe: Connection): """The main loop of the game in machine learning mode This loop is run in a seperate process, and it communicates the main process and the machine learning process with pipes. The execution order in the loops: 1. Send the SceneInfo to the machine learning process and the main process. 2. Wait for the keyframe (During this period, machine learning process can process the SceneInfo and generate the GameInstruction.) 3. Check if there has a GameInstruction to receive. If it has, receive the instruction. Otherwise, generate a dummy one. 4. Pass the GameInstruction to the game and update the scene (and frame no.) 5. If the game is over or passed, send the SceneInfo containing the game status to the both processes, and reset the game. 6. Back to 1. @param fps Specify the updating rate of the game @param level Specify the level of the game @param instruct_pipe The receving-end of the GameInstruction from the ml process @param scene_info_pipe The sending-end of the SceneInfo to the ml process @param main_pipe The sending-end of the SceneInfo to the main process """ def recv_instruction(): if instruct_pipe.poll(): instruction = instruct_pipe.recv() # Pass the exception to the main process if isinstance(instruction, ExceptionMessage): main_pipe.send((instruction, None)) if not isinstance(instruction, GameInstruction): return GameInstruction(-1, gamecore.ACTION_NONE) if instruction.command != gamecore.ACTION_LEFT and \ instruction.command != gamecore.ACTION_RIGHT: instruction.command = gamecore.ACTION_NONE else: return GameInstruction(-1, gamecore.ACTION_NONE) return instruction def wait_ml_process_ready(): while True: ready_instruct = instruct_pipe.recv() # Pass the exception to the main process if isinstance(ready_instruct, ExceptionMessage): main_pipe.send((ready_instruct, None)) return if isinstance(ready_instruct, GameInstruction) and \ ready_instruct.command == GameInstruction.CMD_READY: return scene = gamecore.Scene(level, False) scene_info = scene.fill_scene_info_obj(SceneInfo()) wait_ml_process_ready() # Set the first tick self._clock.tick(fps) while True: scene_info_pipe.send(scene_info) self._clock.tick(fps) instruction = recv_instruction() main_pipe.send((scene_info, instruction)) game_status = scene.update(instruction.command) scene_info = scene.fill_scene_info_obj(SceneInfo()) if game_status == gamecore.GAME_OVER_MSG or \ game_status == gamecore.GAME_PASS_MSG: scene_info_pipe.send(scene_info) main_pipe.send((scene_info, None)) # TODO Wait the ml process before reset the scene scene.reset() scene_info = scene.fill_scene_info_obj(SceneInfo())
def _play_proc(self, msg_dict: dict, pipe: Connection): """Player process.""" # Open the file to play. try: with open_file(**msg_dict) as fileobj: # Put the file info in msg_dict. msg_dict['info'] = str(fileobj) msg_dict['length'] = fileobj.length state = None # Open an audio output device that can handle the data # from fileobj. device = open_device(fileobj, 'w', **msg_dict) msg_out(f"\nFile: {repr(fileobj)}\n") msg_out(f"\nDevice: {repr(device)}\n") try: # Set the default number of loops to infinite. fileobj.loops = msg_dict.get('loops', -1) # Initialize variable. buf = b'\x00' * device.buffer_size # Loop until stopped. while msg_dict.get('playing', True): # Stop if the read buffer is empty or player is # not paused. if not (buf or msg_dict.get('paused', False)): break # Print the stream position. if msg_dict.get('show_position', False): print_position(fileobj) # Keep playing if not paused. if not msg_dict.get('paused', False): # Re-open the device after comming out of # paused state. if device.closed: device = open_device( fileobj, 'w', cached=True, **msg_dict ) # Read the next buffer full of data. try: buf = fileobj.readline() except KeyboardInterrupt: break # Sometimes a read needs to be made before the # length is available. if not msg_dict['length']: msg_dict['length'] = fileobj.length if device._rate != fileobj._rate \ and fileobj._rate != 0: # Convert the input sample rate to that of # the output device. buf, state = audioop.ratecv( buf, fileobj._width, fileobj._channels, fileobj._rate, int(device._rate), state ) # Filler for end of partial buffer to elminiate # end of audio noise. if type(buf) == bytes: filler = b'\x00' * \ (device.buffer_size - len(buf)) else: filler = '' # Write buf. try: _ = device.write(buf + filler) except KeyboardInterrupt: break else: # Close the device when paused and sleep to # open the audio for another process and # save cpu cycles. if not device.closed: device.close() sleep(0.05) # Write a buffer of null bytes so the audio # system can keep its buffer full. # device.write(b'\x00' * device.buffer_size) # Get and process any commands from the parent process. if pipe.poll(): # Get the data into temp. command = pipe.recv() if 'getposition' in command: pipe.send(fileobj.position) elif 'setposition' in command: fileobj.position = command['setposition'] elif 'getloops' in command: pipe.send(fileobj.loops) elif 'setloops' in command: fileobj.loops = command['setloops'] elif 'getloopcount' in command: pipe.send(fileobj.loop_count) elif 'getlength' in command: pipe.send(fileobj.length) except Exception as err: print(err) finally: if not device.closed: device.close() except IOError as err: msg_dict['error'] = err msg_dict['info'] = '' msg_dict['length'] = 0 print(err) finally: try: # Set playing to False for the parent. msg_dict['playing'] = False except BrokenPipeError: pass
def digest_worker(id: int, empty_queue_and_stop_flag: Event, stop_immediately_flag: Event, protein_queue: Queue, log_connection: Connection, unprocessible_log_connection: Connection, statistics: Array, enzyme: DigestEnzyme, database_url: str): log_connection.send("digest worker {} is online".format(id)) engine = create_engine(database_url, pool_size=1, max_overflow=0, pool_timeout=3600) SessionClass = sessionmaker(bind=engine, autoflush=False) # Let the process run until empty_queue_and_stop_flag is true and protein_queue is empty or stop_immediately_flag is true. while (not empty_queue_and_stop_flag.is_set() or not protein_queue.empty() ) and not stop_immediately_flag.is_set(): try: # Try to get a protein from the queue, timeout is 2 seconds protein, protein_merges = protein_queue.get(True, 5) # Variables for loop control unsolvable_errors = 0 try_transaction_again = True while try_transaction_again: session = SessionClass() number_of_new_peptides = 0 try: count_protein = False # Check if the Protein exists by its accession or secondary accessions accessions = [protein.accession] + [ protein_merge.source_accession for protein_merge in protein_merges ] existing_protein = session.query(Protein).filter( Protein.accession.in_(accessions)).first() if existing_protein: need_commit, number_of_new_peptides = Digestion.update_protein( session, protein, existing_protein, protein_merges, enzyme) if need_commit: session.commit() else: session.rollback() else: number_of_new_peptides = Digestion.create_protein( session, protein, protein_merges, enzyme) session.commit() count_protein = True # Commit was successfully stop while-loop and add statistics try_transaction_again = False if count_protein: statistics[0] += 1 statistics[1] += number_of_new_peptides except: # Rollback session session.rollback() # Remove all peptides from protein protein.peptides = [] # Try again after 5 (first try) and 10 (second try) + a random number between 0 and 5 (both including) seconds maybe some blocking transactions can pass so this transaction will successfully finish on the next try. # If this is the third time an unsolvable error occures give up and log the error. if unsolvable_errors < 2: unsolvable_errors += 1 time.sleep(5 * unsolvable_errors + random.randint(0, 5)) # Log the error on the third try and put the protein in unprocessible queue else: exception = sys.exc_info()[1] log_connection.send( "Exception on protein {}, see:\n{}".format( protein.accession, exception)) unprocessible_log_connection.send( protein.to_fasta_entry(protein_merges)) statistics[2] += 1 try_transaction_again = False finally: session.close() # Catch queue.Empty which is thrown when protein_queue.get() timed out except Empty: pass log_connection.send("digest worker {} is stopping".format(id)) log_connection.close() unprocessible_log_connection.close()
def prepare_sync_send(cls, ctrl_connection: Connection): ctrl_connection.send("Initializing stream...") usrp.setup_stream() ret = usrp.start_stream(0) ctrl_connection.send("Initialize stream:{0}".format(ret)) return ret
def prepare_sync_receive(cls, ctrl_connection: Connection): ret = rtlsdr.reset_buffer() ctrl_connection.send("RESET_BUFFER:" + str(ret)) return ret
def start_stream_message(conn: Connection): add_video_message = ModuleMessage(scm, "start_stream", "contents") conn.send(add_video_message)