def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = usrp.open(device_identifier)
ctrl_connection.send("OPEN:" + str(ret))
success = ret == 0
if success:
device_repr = usrp.get_device_representation()
ctrl_connection.send(device_repr)
return success
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = hackrf.setup(device_identifier)
msg = "SETUP"
if device_identifier:
msg += " ({})".format(device_identifier)
msg += ": "+str(ret)
ctrl_connection.send(msg)
return ret == 0
def shutdown_device(cls, ctrl_conn: Connection):
logger.debug("HackRF: closing device")
ret = hackrf.close()
ctrl_conn.send("CLOSE:" + str(ret))
ret = hackrf.exit()
ctrl_conn.send("EXIT:" + str(ret))
return True
def prepare_sync_send(cls, ctrl_connection: Connection):
try:
cls.pyaudio_stream = cls.pyaudio_handle.open(format=pyaudio.paFloat32,
channels=2,
rate=cls.SAMPLE_RATE,
output=True)
ctrl_connection.send("Successfully started pyaudio stream")
return 0
except Exception as e:
logger.exception(e)
ctrl_connection.send("Failed to start pyaudio stream")
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = limesdr.open()
ctrl_connection.send("OPEN:" + str(ret))
limesdr.disable_all_channels()
if ret != 0:
return False
ret = limesdr.init()
ctrl_connection.send("INIT:" + str(ret))
return ret == 0
def prepare_sync_receive(cls, ctrl_connection: Connection):
try:
cls.pyaudio_stream = cls.pyaudio_handle.open(format=pyaudio.paFloat32,
channels=2,
rate=cls.SAMPLE_RATE,
input=True,
frames_per_buffer=cls.CHUNK_SIZE)
ctrl_connection.send("Successfully started pyaudio stream")
return 0
except Exception as e:
logger.exception(e)
ctrl_connection.send("Failed to start pyaudio stream")
def generate_data(connection: Connection, num_samples=32768):
frequency = 0.1
divisor = 200
pos = 0
while True:
result = np.zeros(num_samples, dtype=np.complex64)
result.real = np.cos(2 * np.pi * frequency * np.arange(pos, pos + num_samples))
result.imag = np.sin(2 * np.pi * frequency * np.arange(pos, pos + num_samples))
pos += num_samples
if pos / num_samples >= divisor:
frequency *= 2
if frequency >= 1:
frequency = 0.1
pos = 0
connection.send(result)
def setup_device(cls, ctrl_connection: Connection, device_identifier):
device_identifier = device_identifier if isinstance(device_identifier, str) else ""
try:
device_identifier = re.search("\[.*\]", device_identifier).groups(0)
except (IndexError, AttributeError):
pass
if not device_identifier:
_, uris = plutosdr.scan_devices()
try:
device_identifier = uris[0]
except IndexError:
ctrl_connection.send("Could find a connected PlutoSDR")
return False
ret = plutosdr.open(device_identifier)
ctrl_connection.send("OPEN ({}):{}".format(device_identifier, ret))
return ret == 0
def shutdown_device(cls, ctrl_conn: Connection, is_tx: bool):
if is_tx:
result = hackrf.stop_tx_mode()
ctrl_conn.send("STOP TX MODE:" + str(result))
else:
result = hackrf.stop_rx_mode()
ctrl_conn.send("STOP RX MODE:" + str(result))
result = hackrf.close()
ctrl_conn.send("CLOSE:" + str(result))
result = hackrf.exit()
ctrl_conn.send("EXIT:" + str(result))
return True
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ctrl_connection.send("Initializing pyaudio...")
try:
cls.pyaudio_handle = pyaudio.PyAudio()
ctrl_connection.send("Initialized pyaudio")
return True
except Exception as e:
logger.exception(e)
ctrl_connection.send("Failed to initialize pyaudio")
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters):
ctrl_connection.send("failed to start tx mode")
return False
if cls.ASYNCHRONOUS:
ret = cls.enter_async_send_mode(send_config.get_data_to_send)
else:
ret = cls.prepare_sync_send(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start tx mode")
return False
exit_requested = False
buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
ctrl_connection.send("successfully started tx mode")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
try:
time.sleep(0.5)
except KeyboardInterrupt:
pass
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if not cls.ASYNCHRONOUS:
# Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure
# that the send buffer on the SDR is cleared
time.sleep(0.75)
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(cls.__class__.__name__))
cls.shutdown_device(ctrl_connection, is_tx=True)
ctrl_connection.close()
def init_device(cls, ctrl_connection: Connection, is_tx: bool, parameters: OrderedDict):
if not cls.setup_device(ctrl_connection, device_identifier=parameters["identifier"]):
return False
limesdr.enable_channel(True, is_tx, parameters[cls.Command.SET_CHANNEL_INDEX.name])
limesdr.set_tx(is_tx)
for parameter, value in parameters.items():
cls.process_command((parameter, value), ctrl_connection, is_tx)
antennas = limesdr.get_antenna_list()
ctrl_connection.send("Current normalized gain is {0:.2f}".format(limesdr.get_normalized_gain()))
ctrl_connection.send("Current antenna is {0}".format(antennas[limesdr.get_antenna()]))
ctrl_connection.send("Current chip temperature is {0:.2f}°C".format(limesdr.get_chip_temperature()))
return True
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters):
ctrl_connection.send("failed to start rx mode")
return False
try:
cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name])
except NotImplementedError:
# Many SDRs like HackRF or AirSpy do not need to calculate SYNC_RX_CHUNK_SIZE
# as default values are either fine or given by the hardware
pass
if cls.ASYNCHRONOUS:
ret = cls.enter_async_receive_mode(data_connection, ctrl_connection)
else:
ret = cls.prepare_sync_receive(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start rx mode")
return False
exit_requested = False
ctrl_connection.send("successfully started rx mode")
while not exit_requested:
if cls.ASYNCHRONOUS:
try:
time.sleep(0.25)
except KeyboardInterrupt:
pass
else:
cls.receive_sync(data_connection)
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False)
if result == cls.Command.STOP.name:
exit_requested = True
break
cls.shutdown_device(ctrl_connection, is_tx=False)
data_connection.close()
ctrl_connection.close()
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
usrp.start_stream(cls.READ_SAMPLES)
ctrl_connection.send("Initialized stream")
def _custom_worker_shared_memory(index: int,
env_fn: Callable[[], Env],
pipe: Connection,
parent_pipe: Connection,
shared_memory,
error_queue: Queue,
in_series: int = None):
"""Copied and modified from `gym.vector.async_vector_env`.
Args:
index ([type]): [description]
env_fn ([type]): [description]
pipe ([type]): [description]
parent_pipe ([type]): [description]
shared_memory ([type]): [description]
error_queue ([type]): [description]
Raises:
RuntimeError: [description]
"""
process_name = mp.current_process().name
# print(f"Current process name: {process_name}")
assert shared_memory is not None
env = env_fn()
observation_space = env.observation_space
parent_pipe.close()
def step_fn(actions):
observation, reward, done, info = env.step(actions)
if isinstance(env, VectorEnv):
# Do nothing: Since the env is a VectorEnv, it will automatically
# reset the env if needed in 'step' and return the initial
# observation instead of the final observation.
return observation, reward, done, info
if done:
if info is None:
info = {}
assert isinstance(info, dict)
info[FINAL_STATE_KEY] = observation
observation = env.reset()
return observation, reward, done, info
try:
while True:
command, data = pipe.recv()
# print(f"Worker {index} received command {command}")
if command == Commands.reset:
observation = env.reset()
write_to_shared_memory(index, observation, shared_memory,
observation_space)
pipe.send((None, True))
elif command == Commands.step:
observation, reward, done, info = step_fn(data)
write_to_shared_memory(index, observation, shared_memory,
observation_space)
pipe.send(((None, reward, done, info), True))
elif command == Commands.seed:
env.seed(data)
pipe.send((None, True))
elif command == Commands.close:
pipe.send((None, True))
break
elif command == '_check_observation_space':
pipe.send((data == observation_space, True))
# Below this: added commands.
elif command == Commands.apply:
assert callable(data)
function = data
results = function(env)
pipe.send((results, True))
elif command == Commands.render:
pipe.send(env.render(mode="rgb_array"))
else:
raise RuntimeError('Received unknown command `{0}`. Must '
'be one of {`reset`, `step`, `seed`, `close`, '
'`_check_observation_space`, `apply`}.'.format(command))
except (KeyboardInterrupt, Exception):
error_queue.put((index,) + sys.exc_info()[:2])
pipe.send((None, False))
finally:
env.close()
def enter_async_receive_mode(cls, data_connection: Connection,
ctrl_connection: Connection):
ret = airspy.start_rx(data_connection.send_bytes)
ctrl_connection.send("Start RX MODE:" + str(ret))
def set_state_value(conn: Connection, key, value):
state_message = ModuleMessage("PRGH", "set_state_data", (key, value))
conn.send(state_message)
def prepare_sync_receive(cls, ctrl_connection: Connection):
ret = rtlsdr.reset_buffer()
ctrl_connection.send("RESET_BUFFER:" + str(ret))
return ret == 0
def enter_async_receive_mode(cls, data_connection: Connection,
ctrl_connection: Connection):
ret = hackrf.start_rx_mode(data_connection.send_bytes)
ctrl_connection.send("Start RX MODE:" + str(ret))
return ret
def _run_file_processor(
result_channel: MultiprocessingConnection,
parent_channel: MultiprocessingConnection,
file_path: str,
pickle_dags: bool,
dag_ids: Optional[List[str]],
thread_name: str,
callback_requests: List[CallbackRequest],
) -> None:
"""
Process the given file.
:param result_channel: the connection to use for passing back the result
:type result_channel: multiprocessing.Connection
:param parent_channel: the parent end of the channel to close in the child
:type parent_channel: multiprocessing.Connection
:param file_path: the file to process
:type file_path: str
:param pickle_dags: whether to pickle the DAGs found in the file and
save them to the DB
:type pickle_dags: bool
:param dag_ids: if specified, only examine DAG ID's that are
in this list
:type dag_ids: list[str]
:param thread_name: the name to use for the process that is launched
:type thread_name: str
:param callback_requests: failure callback to execute
:type callback_requests: List[airflow.utils.callback_requests.CallbackRequest]
:return: the process that was launched
:rtype: multiprocessing.Process
"""
# This helper runs in the newly created process
log: logging.Logger = logging.getLogger("airflow.processor")
# Since we share all open FDs from the parent, we need to close the parent side of the pipe here in
# the child, else it won't get closed properly until we exit.
log.info("Closing parent pipe")
parent_channel.close()
del parent_channel
set_context(log, file_path)
setproctitle(f"airflow scheduler - DagFileProcessor {file_path}")
try:
# redirect stdout/stderr to log
with redirect_stdout(StreamLogWriter(
log, logging.INFO)), redirect_stderr(
StreamLogWriter(log,
logging.WARN)), Stats.timer() as timer:
# Re-configure the ORM engine as there are issues with multiple processes
settings.configure_orm()
# Change the thread name to differentiate log lines. This is
# really a separate process, but changing the name of the
# process doesn't work, so changing the thread name instead.
threading.current_thread().name = thread_name
log.info("Started process (PID=%s) to work on %s", os.getpid(),
file_path)
dag_file_processor = DagFileProcessor(dag_ids=dag_ids, log=log)
result: Tuple[int, int] = dag_file_processor.process_file(
file_path=file_path,
pickle_dags=pickle_dags,
callback_requests=callback_requests,
)
result_channel.send(result)
log.info("Processing %s took %.3f seconds", file_path,
timer.duration)
except Exception:
# Log exceptions through the logging framework.
log.exception("Got an exception! Propagating...")
raise
finally:
# We re-initialized the ORM within this Process above so we need to
# tear it down manually here
settings.dispose_orm()
result_channel.close()
def _worker(
parent: connection.Connection,
p: connection.Connection,
env_fn_wrapper: CloudpickleWrapper,
obs_bufs: Optional[Union[dict, tuple, ShArray]] = None,
) -> None:
def _encode_obs(obs: Union[dict, tuple, np.ndarray],
buffer: Union[dict, tuple, ShArray]) -> None:
if isinstance(obs, np.ndarray) and isinstance(buffer, ShArray):
buffer.save(obs)
elif isinstance(obs, tuple) and isinstance(buffer, tuple):
for o, b in zip(obs, buffer):
_encode_obs(o, b)
elif isinstance(obs, dict) and isinstance(buffer, dict):
for k in obs.keys():
_encode_obs(obs[k], buffer[k])
return None
parent.close()
env = env_fn_wrapper.data()
try:
while True:
try:
cmd, data = p.recv()
except EOFError: # the pipe has been closed
p.close()
break
if cmd == "step":
obs, reward, done, info = env.step(data)
if obs_bufs is not None:
_encode_obs(obs, obs_bufs)
obs = None
p.send((obs, reward, done, info))
elif cmd == "reset":
obs = env.reset()
if obs_bufs is not None:
_encode_obs(obs, obs_bufs)
obs = None
p.send(obs)
elif cmd == "close":
p.send(env.close())
p.close()
break
elif cmd == "render":
p.send(env.render(**data) if hasattr(env, "render") else None)
elif cmd == "seed":
p.send(env.seed(data) if hasattr(env, "seed") else None)
elif cmd == "getattr":
p.send(getattr(env, data) if hasattr(env, data) else None)
else:
p.close()
raise NotImplementedError
except KeyboardInterrupt:
p.close()
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
limesdr.setup_stream(cls.SEND_FIFO_SIZE)
ret = limesdr.start_stream()
ctrl_connection.send("Initialize stream:{0}".format(ret))
return ret
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
ret = usrp.start_stream(0)
ctrl_connection.send("Initialize stream:{0}".format(ret))
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing PlutoSDR...")
ret = plutosdr.setup_tx(cls.SYNC_TX_CHUNK_SIZE // 2)
return ret
def enter_async_receive_mode(cls, data_connection: Connection, ctrl_connection: Connection):
ret = hackrf.start_rx_mode(data_connection.send_bytes)
ctrl_connection.send("Start RX MODE:" + str(ret))
return ret
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing BladeRF...")
ret = bladerf.prepare_sync()
return ret
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = hackrf.setup()
ctrl_connection.send("SETUP:" + str(ret))
return ret == 0
def run(node_config: List[int], num_events: int, duration: int,
windows: str, agg_functions: str, is_single_node: bool = False,
is_fixed_events: bool = False, process_log_dir_pipe: Connection = None):
assert_valid_node_config(node_config)
num_nodes = sum(node_config) + 1
log_dir = get_log_dir(num_nodes, num_events, duration)
log_dir = create_log_dir(log_dir)
if process_log_dir_pipe is not None:
sys.stdout = open(os.path.join(log_dir, "main.log"), "w")
print(f"Writing logs to {log_dir}\n")
all_hosts = get_hosts_for_run(node_config)
all_hosts.insert(0, [ROOT_HOST])
flat_hosts = [host for level in all_hosts for host in level]
print(f"All hosts ({len(flat_hosts)}): {' '.join(flat_hosts)}")
port_iter = iter(ALL_PORTS)
print("Uploading benchmark arguments on all nodes...")
# Upload for root node
named_hosts = [(ROOT_HOST, "root")]
upload_root_params(len(all_hosts[1]), windows, agg_functions, is_single_node)
# Upload for intermediate nodes
for level_id, level_hosts in enumerate(all_hosts[1:-2], 1):
parent_nodes = all_hosts[level_id - 1]
num_parents = len(parent_nodes)
num_children = len(all_hosts[level_id + 1])
assert num_children % len(level_hosts) == 0
num_children_per_node = num_children // len(level_hosts)
for node_id, level_host in enumerate(level_hosts):
parent_host = parent_nodes[node_id % num_parents]
node_level_id = (10 * level_id) + node_id
upload_intermediate_params(level_host, node_level_id, num_children_per_node, parent_host, is_single_node)
named_hosts.append((level_host, f"merger-{node_level_id}"))
parent_nodes = all_hosts[-3]
child_hosts = all_hosts[-2]
stream_hosts = all_hosts[-1]
num_parents = len(parent_nodes)
num_children = len(child_hosts)
num_streams = len(stream_hosts)
# # Upload for child nodes
# num_streams_per_child = num_streams // num_children
# for child_id, child_host in enumerate(child_hosts):
# parent_host = parent_nodes[child_id % num_parents]
# upload_child_params(child_host, parent_host, child_id, num_streams_per_child, is_single_node)
# named_hosts.append((child_host, f"child-{child_id}"))
# Upload for child and stream nodes
num_streams_per_child = num_streams // num_children
for child_id, child_host in enumerate(child_hosts):
parent_host = parent_nodes[child_id % num_parents]
child_port = next(port_iter)
upload_child_params(child_host, parent_host, child_port, child_id, num_streams_per_child, is_single_node)
named_hosts.append((child_host, f"child-{child_id}"))
upload_stream_params(child_host, child_id, child_port, num_events, duration,
agg_functions, is_fixed_events)
named_hosts.append((child_host, f"stream-{child_id}"))
# TODO
# # Upload for stream nodes
# for stream_id, stream_host in enumerate(stream_hosts):
# parent_host = child_hosts[stream_id % num_children]
# upload_stream_params(stream_host, stream_id, parent_host, num_events, duration,
# agg_functions, is_fixed_events)
# named_hosts.append((stream_host, f"stream-{stream_id}"))
print("Starting `run.sh` on all nodes.\n")
max_run_duration = duration + 30
threads = []
for host, name in named_hosts:
mode = ""
if name.startswith("child"):
mode = "CHILD"
if name.startswith("stream"):
mode = "STREAM"
thread = Thread(target=run_host,
args=(host, name, log_dir, max_run_duration, mode),
name=f"thread-{name}")
thread.start()
threads.append(thread)
# Give nodes time to start running
time.sleep(5)
check_complete(max_run_duration, flat_hosts, complete_check_fn)
kill_command = "pkill -9 -f /home/hadoop/benson/openjdk12/bin/java"
print("Ending script by killing all PIDs...")
for host in flat_hosts:
ssh_command(host, kill_command)
for thread in threads:
thread.join()
print("Joined all run threads.")
if process_log_dir_pipe is not None:
process_log_dir_pipe.send(log_dir)
return log_dir
def env_worker_multiple_envs(conn: Connection, **env_kwargs):
envs = {}
def get_env(field_id):
if field_id in envs:
return envs[field_id]
else:
gameplay = Ccreate_start_conditions()
env = Haxball(gameplay=gameplay, **env_kwargs)
envs[field_id] = env
return env
while True:
cmd, data = conn.recv()
if cmd == 'step':
obss = []
rews = []
dones = []
infos = []
field_ids, a1s, a2s = zip(*data)
players_i = []
for i, field_id in enumerate(field_ids):
field_id = field_ids[i]
a1 = a1s[i]
a2 = a2s[i]
# rf1 = rf1s[i]
# rf2 = rf2s[i]
players_i.append(field_id * 2)
players_i.append(field_id * 2 + 1)
env = get_env(field_id)
# Le azioni non sono immediate
# steps_to_wait = max(1, np.random.poisson(lam=1))
env.step_physics(1)
env.step_async(a1, red_team=True)
env.step_async(a2, red_team=False)
# Le azioni sfalsate
delay = int(np.round(np.clip(np.random.normal(8, 1), 5, 11)))
if random.random() < 0.5:
pause1 = delay // 2
pause2 = delay - pause1
else:
pause2 = delay // 2
pause1 = delay - pause2
# env.step_physics(pause1)
env.step_physics(pause1 + pause2)
# env.step_async(a2, red_team=False)
# env.step_physics(1)
# Le misure di obs non sono immediate
# steps_to_wait = max(1, np.random.poisson(lam=2))
# steps_to_wait = 1
# steps_to_wait = max(1, np.random.poisson(lam=1))
# env.step_physics(steps_to_wait)
is_done = False
for red_team in (True, False):
obs, rew, done, info = env.step_wait(red_team=red_team)
obss.append(obs)
rews.append(rew)
dones.append(done)
info['field_id'] = field_id
info[
'players_i'] = field_id * 2 if red_team else field_id * 2 + 1
infos.append(info)
is_done |= done
# if red_team and not done:
# env.step_physics(pause2)
if is_done:
env.reset()
res = field_ids, players_i, np.array(obss), np.array(
rews), np.array(dones), np.array(infos)
conn.send(res)
elif cmd == 'reset':
field_id = data
env = get_env(field_id)
ob = env.reset()
conn.send([ob, ob])
elif cmd == 'render':
field_id = data
env = get_env(field_id)
res = env.render(mode='rgb_array')
conn.send(res)
elif cmd == 'close':
conn.close()
break
elif cmd == 'get_spaces_spec':
field_id = data
env = get_env(field_id)
conn.send((env.observation_space, env.action_space, env.spec))
else:
raise NotImplementedError
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing PlutoSDR..")
ret = plutosdr.setup_rx(cls.SYNC_RX_CHUNK_SIZE)
return ret
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
usrp.start_stream(cls.READ_SAMPLES)
ctrl_connection.send("Initialized stream")
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing PlutoSDR...")
ret = plutosdr.setup_tx(cls.SYNC_TX_CHUNK_SIZE // 2)
return ret
def wrapped_run(pipe: Connection):
result = job.run()
pipe.send(result)
def mailer_process(
*,
queue: mp.Queue,
error_channel: mpc.Connection,
quit_event: mp.Event,
rate_limiter: RateLimiter = None,
rate_meter: RateMeter = None,
):
"""
Main function of the processes responsible for sending email messages.
This process pulls `(message: EmailMessage, event: CampaignSentEvent)`
tuples from the work queue, tries to send the message and saves the result
on the `event`.
Whenever an unexpected error happens (i.e. which does not seem to be linked
to a particular recipient), the process signals the error on the
`error_channel` by sending the campaign id.
It monitors its own connection to the mail service, and resets it every
`nuntius.app_settings.MAX_MESSAGES_PER_CONNECTION` messages.
:param queue: The work queue on which tuples (EmailMessage, CampaignSentEvent) are received.
:type queue: class:`multiprocessing.Queue`
:param error_channel: channel on which campaign ids of failing campaigns are sent.
:type error_channel: class:`multiprocessing.connection.Connection`
:param quit_event: event that may be used by the main process to signal to senders they need to quit
:type quit_event: class:`multiprocessing.Event`
:param rate_limiter: rate limiter to limit the rate of message sendings.
:type rate_limiter: class:`nuntius.utils.processes.RateLimiter`
:param rate_meter: rate meter to allow measuring the sending speed
:type rate_meter: class:`nuntius.utils.processes.RateMeter`
"""
message: EmailMessage
sent_event_id: int
try:
with ConnectionManager(quit_event) as connection_manager:
while True:
# the timeout allows the loop to start again every few seconds so that the
# quit_event is checked and the process can quit if it has to.
message, sent_event_id = get_from_queue_or_quit(
queue,
event=quit_event,
polling_period=app_settings.POLLING_INTERVAL,
)
sent_event_qs = CampaignSentEvent.objects.filter(id=sent_event_id)
# messages generated by the campaign manager should only have one recipient
email = message.to[0]
# rate limit just before sending
if rate_limiter:
rate_limiter.take()
try:
connection_manager.send_message(message)
except (smtplib.SMTPRecipientsRefused, AnymailRecipientsRefused):
# exceptions linked to a specific recipient need not stop the sending
CampaignSentEvent.objects.filter(id=sent_event_id).update(
result=CampaignSentStatusType.BLOCKED
)
except GracefulExit:
raise
except Exception:
campaign = Campaign.objects.get(campaignsentevent__id=sent_event_id)
error_channel.send(campaign.id)
logger.error(
_("Error while sending email for campaign %(campaign)s")
% repr(campaign),
exc_info=True,
)
else:
if rate_meter:
rate_meter.count_up()
if hasattr(message, "anymail_status"):
if message.anymail_status.recipients[email].status in [
"invalid",
"rejected",
"failed",
]:
sent_event_qs.update(result=CampaignSentStatusType.REJECTED)
else:
sent_event_qs.update(
result=CampaignSentStatusType.UNKNOWN,
esp_message_id=message.anymail_status.recipients[
email
].message_id,
)
else:
sent_event_qs.update(result=CampaignSentStatusType.UNKNOWN)
except GracefulExit:
return
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = airspy.open()
ctrl_connection.send("OPEN:" + str(ret))
return ret == 0
def pusher_process(
*,
queue: mp.Queue,
error_channel: mpc.Connection,
quit_event: mp.Event,
rate_limiter: RateLimiter = None,
rate_meter: RateMeter = None,
):
"""
Main function of the processes responsible for sending push notifications.
This process pulls `(push_notification, event: PushCampaignSentEvent)`
tuples from the work queue, tries to send the push and saves the result
on the `event`.
Whenever an unexpected error happens (i.e. which does not seem to be linked
to a particular recipient), the process signals the error on the
`error_channel` by sending the campaign id.
It monitors its own connection to the mail service, and resets it every
`nuntius.app_settings.MAX_MESSAGES_PER_CONNECTION` messages.
:param queue: The work queue on which tuples (push_notification, CampaignSentEvent) are received.
:type queue: class:`multiprocessing.Queue`
:param error_channel: channel on which campaign ids of failing campaigns are sent.
:type error_channel: class:`multiprocessing.connection.Connection`
:param quit_event: event that may be used by the main process to signal to senders they need to quit
:type quit_event: class:`multiprocessing.Event`
:param rate_limiter: rate limiter to limit the rate of message sendings.
:type rate_limiter: class:`nuntius.utils.processes.RateLimiter`
:param rate_meter: rate meter to allow measuring the sending speed
:type rate_meter: class:`nuntius.utils.processes.RateMeter`
"""
notification: dict
push_sent_event_id: int
try:
with PushManager(quit_event) as push_manager:
while True:
# the timeout allows the loop to start again every few seconds so that the
# quit_event is checked and the process can quit if it has to.
notification, push_sent_event_id = get_from_queue_or_quit(
queue,
event=quit_event,
polling_period=app_settings.POLLING_INTERVAL,
)
# rate limit just before sending
if rate_limiter:
rate_limiter.take()
try:
push_sent_event = PushCampaignSentEvent.objects.get(
id=push_sent_event_id
)
push_manager.push(notification, push_sent_event)
except GracefulExit:
raise
except PushCampaignSentEvent.DoesNotExist:
logger.error(
_(
f"Push campaign sent event with id '{push_sent_event_id}' not found"
),
exc_info=True,
)
except Exception:
push_campaign = PushCampaign.objects.get(
pushcampaignsentevent__id=push_sent_event_id
)
error_channel.send(push_campaign.id)
logger.error(
_(
f"Error while pushing notification for campaign {repr(push_campaign)}"
),
exc_info=True,
)
else:
if rate_meter:
rate_meter.count_up()
except GracefulExit:
return
def setup_device(cls, ctrl_connection: Connection, device_identifier):
# identifier gets set in self.receive_process_arguments
device_number = int(device_identifier)
ret = rtlsdr.open(device_number)
ctrl_connection.send("OPEN:" + str(ret))
return ret == 0
def ret_through_conn(*args, fn: Callable, send_conn: Connection,
**kwargs) -> None:
send_conn.send(fn(*args, **kwargs))
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = airspy.open()
ctrl_connection.send("OPEN:" + str(ret))
return ret == 0
def evolve(in_con:MConnection, out_con: MConnection, returnq: MQueue, settings: EnhanceModelSettings,
optimal_primary_list: List[Gear], optimal_cost, cum_cost, secondaries=None, ev_set:EvolveSettings=None):
if ev_set is None:
ev_set = EvolveSettings()
if secondaries is None:
secondaries = settings[settings.P_FAIL_STACKER_SECONDARY]
num_fs = ev_set.num_fs
population_size = ev_set.pop_size
ultra_elitism = ev_set.ultra_elite
num_elites = ev_set.num_elites
brood_size = ev_set.brood_size
seent = set()
this_seent = []
trait_dominance = ev_set.trait_dom
mutation_rate = ev_set.mutation_rate
extinction_epoch = ev_set.extinction_epoch
max_mutation = ev_set.max_mutation
f_fitness = fitness_funcs[ev_set.fitness_function]
oppressive_mode = ev_set.oppressive_mode
oppress_suprem = ev_set.penalize_supremacy
best = cum_cost
best_fsl = None
lb = 0
def reg_prune(x: FailStackList):
# sig = (x.starting_pos, x.secondary_gear, *x.secondary_map[:-1])
sig = (secondaries.index(x.secondary_gear), *x.get_gnome())
if sig in seent:
return False
else:
seent.add(sig)
this_seent.append(sig) # Send this signature to the other processes
return True
def check_pruned(x: FailStackList):
sig = (secondaries.index(x.secondary_gear), *x.get_gnome())
return sig not in seent
def accept_prune(x):
return True
check_2 = accept_prune
check = accept_prune
if oppress_suprem:
check_2 = reg_prune
check = check_pruned
if oppressive_mode:
check = reg_prune
check_2 = reg_prune
def get_randoms(size_):
retlist = [
FailStackList(settings, choice(secondaries), optimal_primary_list, optimal_cost, cum_cost, num_fs=num_fs) for _ in range(0, size_)]
[p.generate_secondary_map(randint(10, 60)) for p in retlist]
return retlist
def mutate(new_indiv):
this_max_mutation = int(ceil(min(lb / 4, max_mutation)))
for i, v in enumerate(new_indiv.secondary_map[:-1]):
if random() < mutation_rate:
new_v = v + randint(-this_max_mutation, this_max_mutation)
new_indiv.secondary_map[i] = max(1, new_v)
if random() < mutation_rate:
new_s = new_indiv.starting_pos + randint(-this_max_mutation, this_max_mutation)
new_indiv.starting_pos = min(max(5, new_s), 60)
if random() < mutation_rate:
new_indiv.secondary_gear = choice(secondaries)
new_indiv.secondary_map[-1] = 300
best_fitness = fitness_func(optimal_cost, optimal_cost, cum_cost, cum_cost)
this_brood_size = 0
population = get_randoms(population_size)
epoch_mode = False
while True:
if lb > extinction_epoch:
seent = set()
population = get_randoms(population_size)
lb = 0
[p.evaluate_map() for p in population]
#pop_costs = best / numpy.array([f.fs_cum_cost for f in population]) # Bigger is better
#fitness = numpy.sum(pop_costs, axis=1)
pop_costs = numpy.array([f_fitness(f.fs_cost, optimal_cost, f.fs_cum_cost, cum_cost) for f in population]) # Bigger is better
fitness = pop_costs
sort_order = numpy.argsort(fitness, kind='mergesort')
#for i in range(0, min(lb-15, brood_size)):
# bad_fsl = population[sort_order[i]]
# check_2(bad_fsl)
if oppress_suprem:
epoch_mode = lb > 5
if epoch_mode:
for i in population[:this_brood_size]:
check_2(i)
brood_size = max(20, brood_size-lb)
this_best_fitness = fitness[sort_order[-1]]
if this_best_fitness > best_fitness:
best_fitness = this_best_fitness
best_fsl = population[sort_order[-1]]
returnq.put((this_best_fitness, lb, (secondaries.index(best_fsl.secondary_gear), *best_fsl.get_gnome())), block=True)
lb = 0
#best = numpy.min([best, best_fsl.fs_cum_cost], axis=0)
new_pop = []
others_seent = []
try:
while in_con.poll():
others_seent.extend(in_con.recv())
except EOFError: # Pipe broken: terminate loop
out_con.close()
return
except BrokenPipeError:
out_con.close()
return
for i in others_seent:
this_len = len(seent)
seent.add(tuple(i))
if len(seent) > this_len:
this_seent.append(i)
for i in range(0, brood_size):
breeder1 = choice(sort_order[-num_elites:])
breeder1 = population[breeder1]
if not epoch_mode and (best_fsl is not None) and (random() < ultra_elitism):
breeder2 = best_fsl
else:
if random() > (lb * 0.1):
breeder2 = choice(sort_order[-num_elites:])
breeder2 = population[breeder2]
else:
breeder2 = choice(sort_order)
breeder2 = population[breeder2]
offspring = FailStackList(settings, choice([breeder1.secondary_gear, breeder2.secondary_gear]), optimal_primary_list,
optimal_cost, cum_cost, num_fs=num_fs)
offspring.secondary_map = breeder1.secondary_map.copy() # this gets overwritten anyway
for i in range(min(len(breeder1.secondary_map), len(breeder2.secondary_map))):
if random() < trait_dominance:
if random() < 0.5:
offspring.secondary_map[i] = breeder1.secondary_map[i]
else:
offspring.secondary_map[i] = breeder2.secondary_map[i]
else:
offspring.secondary_map[i] = int(round((breeder1.secondary_map[i] + breeder2.secondary_map[i]) / 2.0))
if random() < trait_dominance:
if random() < 0.5:
offspring.starting_pos = breeder1.starting_pos
else:
offspring.starting_pos = breeder2.starting_pos
else:
offspring.starting_pos = int(round((breeder1.starting_pos + breeder2.starting_pos) / 2.0))
offspring.secondary_map[-1] = 300
mutate(offspring)
if check(offspring):
new_pop.append(offspring)
this_brood_size = len(new_pop)
new_pop.extend(get_randoms(population_size-len(new_pop)))
population = new_pop
if len(this_seent) > 0:
out_con.send(this_seent)
this_seent = []
lb += 1
def ConnSend(connection:Connection, msg): connection.send(msg)
def setup_device(cls, ctrl_connection: Connection, device_identifier):
ret = hackrf.setup()
ctrl_connection.send("SETUP:" + str(ret))
return ret == 0
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
limesdr.setup_stream(cls.SEND_FIFO_SIZE)
ret = limesdr.start_stream()
ctrl_connection.send("Initialize stream:{0}".format(ret))
return ret
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
return usrp.start_stream(cls.SYNC_RX_CHUNK_SIZE)
def setup_device(cls, ctrl_connection: Connection, device_identifier):
# identifier gets set in self.receive_process_arguments
device_number = int(device_identifier)
ret = rtlsdr.open(device_number)
ctrl_connection.send("OPEN (#{}):{}".format(device_number, ret))
return ret == 0
def terminal_process(termtype: str, w: int, h: int, pw: int, ph: int,
subproc_pipe: Connection):
"""
Avoid Python curses singleton bug by stuffing Terminal in a subprocess
and proxying calls/responses via Pipe
"""
subproc_term = SubprocessTerminal(termtype, w, h, pw, ph)
while True:
try:
given_attr, args, kwargs = subproc_pipe.recv()
except KeyboardInterrupt:
return
if debug_term:
log.debug(f'proxy received: {given_attr}, {args!r}, '
f'{kwargs!r}')
# exit sentinel
if given_attr is None:
if debug_term:
log.debug(f'term={subproc_term}/pid={os.getpid()} exit')
break
# special attribute -- a context manager, enter it naturally, exit
# unnaturally (even, prematurely), with the exit value ready for
# our client side, this is only possible because blessed doesn't
# use any state or time-sensitive values, only terminal sequences,
# and these CM's are the only ones without side-effects.
if given_attr.startswith('!CTX'):
# here, we feel the real punishment of side-effects...
sideeffect_stream = subproc_term.stream.getvalue()
assert not sideeffect_stream, ('should be empty',
sideeffect_stream)
given_attr = given_attr[len('!CTX'):]
if debug_term:
log.debug(f'context attr: {given_attr}')
with getattr(subproc_term, given_attr)(*args, **kwargs) \
as enter_result:
enter_side_effect = subproc_term.stream.getvalue()
subproc_term.stream.truncate(0)
subproc_term.stream.seek(0)
if debug_term:
log.debug('enter_result, enter_side_effect = '
f'{enter_result!r}, {enter_side_effect!r}')
subproc_pipe.send((enter_result, enter_side_effect))
exit_side_effect = subproc_term.stream.getvalue()
subproc_term.stream.truncate(0)
subproc_term.stream.seek(0)
subproc_pipe.send(exit_side_effect)
elif given_attr.startswith('!CALL'):
given_attr = given_attr[len('!CALL'):]
matching_attr = getattr(subproc_term, given_attr)
if debug_term:
log.debug(f'callable attr: {given_attr}')
subproc_pipe.send(matching_attr(*args, **kwargs))
else:
if debug_term:
log.debug(f'attr: {given_attr}')
assert len(args) == len(kwargs) == 0, (args, kwargs)
matching_attr = getattr(subproc_term, given_attr)
if debug_term:
log.debug(f'value: {matching_attr!r}')
subproc_pipe.send(matching_attr)
def game_loop(self, fps: int, level: int, \
instruct_pipe: Connection, scene_info_pipe: Connection, main_pipe: Connection):
"""The main loop of the game in machine learning mode
This loop is run in a seperate process, and it communicates the main process
and the machine learning process with pipes.
The execution order in the loops:
1. Send the SceneInfo to the machine learning process and the main process.
2. Wait for the keyframe (During this period, machine learning process can
process the SceneInfo and generate the GameInstruction.)
3. Check if there has a GameInstruction to receive. If it has, receive the
instruction. Otherwise, generate a dummy one.
4. Pass the GameInstruction to the game and update the scene (and frame no.)
5. If the game is over or passed, send the SceneInfo containing the
game status to the both processes, and reset the game.
6. Back to 1.
@param fps Specify the updating rate of the game
@param level Specify the level of the game
@param instruct_pipe The receving-end of the GameInstruction from the ml process
@param scene_info_pipe The sending-end of the SceneInfo to the ml process
@param main_pipe The sending-end of the SceneInfo to the main process
"""
def recv_instruction():
if instruct_pipe.poll():
instruction = instruct_pipe.recv()
# Pass the exception to the main process
if isinstance(instruction, ExceptionMessage):
main_pipe.send((instruction, None))
if not isinstance(instruction, GameInstruction):
return GameInstruction(-1, gamecore.ACTION_NONE)
if instruction.command != gamecore.ACTION_LEFT and \
instruction.command != gamecore.ACTION_RIGHT:
instruction.command = gamecore.ACTION_NONE
else:
return GameInstruction(-1, gamecore.ACTION_NONE)
return instruction
def wait_ml_process_ready():
while True:
ready_instruct = instruct_pipe.recv()
# Pass the exception to the main process
if isinstance(ready_instruct, ExceptionMessage):
main_pipe.send((ready_instruct, None))
return
if isinstance(ready_instruct, GameInstruction) and \
ready_instruct.command == GameInstruction.CMD_READY:
return
scene = gamecore.Scene(level, False)
scene_info = scene.fill_scene_info_obj(SceneInfo())
wait_ml_process_ready()
# Set the first tick
self._clock.tick(fps)
while True:
scene_info_pipe.send(scene_info)
self._clock.tick(fps)
instruction = recv_instruction()
main_pipe.send((scene_info, instruction))
game_status = scene.update(instruction.command)
scene_info = scene.fill_scene_info_obj(SceneInfo())
if game_status == gamecore.GAME_OVER_MSG or \
game_status == gamecore.GAME_PASS_MSG:
scene_info_pipe.send(scene_info)
main_pipe.send((scene_info, None))
# TODO Wait the ml process before reset the scene
scene.reset()
scene_info = scene.fill_scene_info_obj(SceneInfo())
def _play_proc(self, msg_dict: dict, pipe: Connection):
"""Player process."""
# Open the file to play.
try:
with open_file(**msg_dict) as fileobj:
# Put the file info in msg_dict.
msg_dict['info'] = str(fileobj)
msg_dict['length'] = fileobj.length
state = None
# Open an audio output device that can handle the data
# from fileobj.
device = open_device(fileobj, 'w', **msg_dict)
msg_out(f"\nFile: {repr(fileobj)}\n")
msg_out(f"\nDevice: {repr(device)}\n")
try:
# Set the default number of loops to infinite.
fileobj.loops = msg_dict.get('loops', -1)
# Initialize variable.
buf = b'\x00' * device.buffer_size
# Loop until stopped.
while msg_dict.get('playing', True):
# Stop if the read buffer is empty or player is
# not paused.
if not (buf or msg_dict.get('paused', False)):
break
# Print the stream position.
if msg_dict.get('show_position', False):
print_position(fileobj)
# Keep playing if not paused.
if not msg_dict.get('paused', False):
# Re-open the device after comming out of
# paused state.
if device.closed:
device = open_device(
fileobj,
'w',
cached=True,
**msg_dict
)
# Read the next buffer full of data.
try:
buf = fileobj.readline()
except KeyboardInterrupt:
break
# Sometimes a read needs to be made before the
# length is available.
if not msg_dict['length']:
msg_dict['length'] = fileobj.length
if device._rate != fileobj._rate \
and fileobj._rate != 0:
# Convert the input sample rate to that of
# the output device.
buf, state = audioop.ratecv(
buf,
fileobj._width,
fileobj._channels,
fileobj._rate,
int(device._rate),
state
)
# Filler for end of partial buffer to elminiate
# end of audio noise.
if type(buf) == bytes:
filler = b'\x00' * \
(device.buffer_size - len(buf))
else:
filler = ''
# Write buf.
try:
_ = device.write(buf + filler)
except KeyboardInterrupt:
break
else:
# Close the device when paused and sleep to
# open the audio for another process and
# save cpu cycles.
if not device.closed:
device.close()
sleep(0.05)
# Write a buffer of null bytes so the audio
# system can keep its buffer full.
# device.write(b'\x00' * device.buffer_size)
# Get and process any commands from the parent process.
if pipe.poll():
# Get the data into temp.
command = pipe.recv()
if 'getposition' in command:
pipe.send(fileobj.position)
elif 'setposition' in command:
fileobj.position = command['setposition']
elif 'getloops' in command:
pipe.send(fileobj.loops)
elif 'setloops' in command:
fileobj.loops = command['setloops']
elif 'getloopcount' in command:
pipe.send(fileobj.loop_count)
elif 'getlength' in command:
pipe.send(fileobj.length)
except Exception as err:
print(err)
finally:
if not device.closed:
device.close()
except IOError as err:
msg_dict['error'] = err
msg_dict['info'] = ''
msg_dict['length'] = 0
print(err)
finally:
try:
# Set playing to False for the parent.
msg_dict['playing'] = False
except BrokenPipeError:
pass
def digest_worker(id: int, empty_queue_and_stop_flag: Event,
stop_immediately_flag: Event, protein_queue: Queue,
log_connection: Connection,
unprocessible_log_connection: Connection,
statistics: Array, enzyme: DigestEnzyme,
database_url: str):
log_connection.send("digest worker {} is online".format(id))
engine = create_engine(database_url,
pool_size=1,
max_overflow=0,
pool_timeout=3600)
SessionClass = sessionmaker(bind=engine, autoflush=False)
# Let the process run until empty_queue_and_stop_flag is true and protein_queue is empty or stop_immediately_flag is true.
while (not empty_queue_and_stop_flag.is_set()
or not protein_queue.empty()
) and not stop_immediately_flag.is_set():
try:
# Try to get a protein from the queue, timeout is 2 seconds
protein, protein_merges = protein_queue.get(True, 5)
# Variables for loop control
unsolvable_errors = 0
try_transaction_again = True
while try_transaction_again:
session = SessionClass()
number_of_new_peptides = 0
try:
count_protein = False
# Check if the Protein exists by its accession or secondary accessions
accessions = [protein.accession] + [
protein_merge.source_accession
for protein_merge in protein_merges
]
existing_protein = session.query(Protein).filter(
Protein.accession.in_(accessions)).first()
if existing_protein:
need_commit, number_of_new_peptides = Digestion.update_protein(
session, protein, existing_protein,
protein_merges, enzyme)
if need_commit:
session.commit()
else:
session.rollback()
else:
number_of_new_peptides = Digestion.create_protein(
session, protein, protein_merges, enzyme)
session.commit()
count_protein = True
# Commit was successfully stop while-loop and add statistics
try_transaction_again = False
if count_protein:
statistics[0] += 1
statistics[1] += number_of_new_peptides
except:
# Rollback session
session.rollback()
# Remove all peptides from protein
protein.peptides = []
# Try again after 5 (first try) and 10 (second try) + a random number between 0 and 5 (both including) seconds maybe some blocking transactions can pass so this transaction will successfully finish on the next try.
# If this is the third time an unsolvable error occures give up and log the error.
if unsolvable_errors < 2:
unsolvable_errors += 1
time.sleep(5 * unsolvable_errors +
random.randint(0, 5))
# Log the error on the third try and put the protein in unprocessible queue
else:
exception = sys.exc_info()[1]
log_connection.send(
"Exception on protein {}, see:\n{}".format(
protein.accession, exception))
unprocessible_log_connection.send(
protein.to_fasta_entry(protein_merges))
statistics[2] += 1
try_transaction_again = False
finally:
session.close()
# Catch queue.Empty which is thrown when protein_queue.get() timed out
except Empty:
pass
log_connection.send("digest worker {} is stopping".format(id))
log_connection.close()
unprocessible_log_connection.close()
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
return usrp.start_stream(cls.SYNC_RX_CHUNK_SIZE)
def prepare_sync_send(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing stream...")
usrp.setup_stream()
ret = usrp.start_stream(0)
ctrl_connection.send("Initialize stream:{0}".format(ret))
return ret
def prepare_sync_receive(cls, ctrl_connection: Connection):
ctrl_connection.send("Initializing PlutoSDR..")
ret = plutosdr.setup_rx(cls.SYNC_RX_CHUNK_SIZE)
return ret
def prepare_sync_receive(cls, ctrl_connection: Connection):
ret = rtlsdr.reset_buffer()
ctrl_connection.send("RESET_BUFFER:" + str(ret))
return ret
def start_stream_message(conn: Connection):
add_video_message = ModuleMessage(scm, "start_stream", "contents")
conn.send(add_video_message)
