def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters):
return False
try:
cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name])
except NotImplementedError:
# Many SDRs like HackRF or AirSpy do not need to calculate READ_SAMPLES
# as default values are either fine or given by the hardware
pass
if cls.ASYNCHRONOUS:
cls.enter_async_receive_mode(data_connection)
else:
cls.prepare_sync_receive(ctrl_connection)
exit_requested = False
while not exit_requested:
if cls.ASYNCHRONOUS:
time.sleep(0.5)
else:
cls.receive_sync(data_connection)
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False)
if result == cls.Command.STOP.name:
exit_requested = True
break
cls.shutdown_device(ctrl_connection)
data_connection.close()
ctrl_connection.close()
def process_taxonomy_merge(id: int, database_url: str, stop_flag: Event,
queue: Queue, log_connection: Connection):
"""
Processes taxonomy merges from a queue.
Parameters
----------
id : int
Just a id for identifying the process in the logs
database_url : str
Database URL, e.g. postgres://username:password@host:port/database
stop_flag : Event
Multiprocessing event to stop the processes
queue : Queue
Multiprocessing queue for queuing the merges
log_connection : Connection
Multiprocessing connection to send logs to the logger
"""
log_connection.send(f"merge worker {id} is online")
database_connection = None
while not stop_flag.is_set() or not queue.empty():
if not database_connection or (database_connection and
database_connection.closed != 0):
database_connection = psycopg2.connect(database_url)
try:
# Take a taxonomy from queue
taxonomy_merges = queue.get(True, 2)
commit_errors = 0
# Try to insert it into the database
while True:
try:
with database_connection:
with database_connection.cursor(
) as database_cursor:
TaxonomyMerge.bulk_insert(
database_cursor, taxonomy_merges)
break
except BaseException as error:
commit_errors += 1
# If there are tries left, sleep between 2 and 5 second and try again
if commit_errors < 3:
time.sleep(random.randint(2, 5))
else:
# Otherwise log the error and proceed with next taxonomy
commaseperated_taxonmy_merges = ", ".join([
f"({taxonomy_merge.source_id},{taxonomy_merge.target_id})"
for taxonomy_merge in taxonomy_merges
])
log_connection.send(
f"taxonomy merge {commaseperated_taxonmy_merges} raises error:\n{error}\n"
)
break
except QueueEmptyError:
# Catch queue empty error (thrown on timeout)
continue
if database_connection and database_connection.closed == 0:
database_connection.close()
log_connection.send(f"merge worker {id} is stopping")
# Close connection to logger
log_connection.close()
def _run_user_proc(user: pwd.struct_passwd, pipe: Connection) -> None:
os.setgid(user.pw_gid)
os.setuid(user.pw_uid)
os.environ[
"DBUS_SESSION_BUS_ADDRESS"
] = f"unix:path=/run/user/{user.pw_uid}/bus"
log.debug(f"Subprocess created with uid={os.getuid()} and gid={os.getgid()}")
try:
temp_noti: Notify.Notification = Notify.Notification.new("No message.")
while not pipe.closed:
log.debug(
f"Blocking for new notifications from {pipe} on subprocess for user {user}"
)
try:
input = pipe.recv()
except KeyboardInterrupt:
input = "QUIT"
if isinstance(input, Notification):
temp_noti.update(input.summary, input.body, input.app_icon)
temp_noti.set_hint_byte("urgency", input.urgency.value)
try:
temp_noti.show()
except GDBusError as e:
log.exception("Failed to show notification.")
elif isinstance(input, str) and input == "QUIT":
log.debug(f"Quit message received. Shutting down.")
pipe.close()
break
except EOFError:
log.debug(f"Pipe closed {pipe} on subprocess for user {user}")
def update(self, conn: Connection, rtsp_url: str, buffer: bool):
"""
Runs the rtspcam thread to grab data and keep the buffer empty.
:param conn: the Pipe to transmit data
:param rtsp_url: the url of the rtspcam.
:param buffer: should the client read frame by frame from the buffer or just grab the latest frame?
:return:
"""
self.log.info(f"Starting video capture client for {rtsp_url}")
cap = VideoCapture(rtsp_url, CAP_FFMPEG)
self.log.info("Started...")
run = True
while run:
if not conn.poll() and not buffer:
self.log.debug("clearing buffer frame")
cap.grab()
continue
rec_dat = conn.recv()
if rec_dat == self.SEND_FRAME:
self.log.debug("Sending next frame to parent process")
return_value, frame = cap.read()
conn.send(frame)
elif rec_dat == self.END_PROCESS:
self.log.debug("Closing connection")
cap.release()
run = False
self.log.info("Camera Connection Closed")
conn.close()
def __chunk_wrapper(
target: Callable[[Any], Any],
target_kwargs: Dict[str, Any],
target_name: str,
connection: Connection,
chunk_index: int,
chunk_count: int,
):
chunk_info = f"{chunk_index + 1}/{chunk_count}"
try:
logger.info(f"[{target_name}] Processing chunk {chunk_info}")
response_data = target(**target_kwargs)
connection.send(response_data)
logger.info(f"[{target_name}] Successfully processed"
f" chunk {chunk_info}")
except Exception as error:
logger.exception(
f"[{target_name}] Error when processing chunk {chunk_info}")
connection.send(error)
finally:
connection.close()
def evaluate(self, weight_path: str, sendPipe: connection.Connection, *args, **kwargs):
'''
功能: 单个模型测试
'''
# 限制内存增长
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
except RuntimeError as e:
print(e)
# 模型配置
model = self.model_preparation()
model = model_cnn_common.model_compile(self.type, model)
# 导入权重
model.load_weights(weight_path, by_name=False)
# 生成器准备
self.generator_preparation()
# 数据准备
self.data_preparation()
# 数据预取
self.testsets = self.testsets.prefetch(
buffer_size=tf.data.AUTOTUNE)
# 模型评估
loss = model.evaluate(
self.testsets,
verbose=1,
return_dict=True)
# 返回
sendPipe.send(loss)
# 对应主进程, 确保管道关闭
sendPipe.close()
def report_progress(conn_out: connection.Connection, prefix: str = '') -> None:
"""Tell the user how much work has been done.
:param prefix: A message to print before the progress prompt, e.g.
'Progress: '.
:param conn_out: A multiprocessing ``Connection`` object from which values
may be read. Each value emitted by the object should be a float between
0 and 1, inclusive, where 0 indicates 0% complete, and 1 represents
100% complete. When a value of 1 is received, this function will
return.
:return: Nothing.
"""
while True:
progress = conn_out.recv()
# \r is carriage return. The other is an ANSI escape code. See:
# https://en.wikipedia.org/wiki/ANSI_escape_code
print(
'\r\033[K' + prefix + f'{progress * 100:.0f}%',
end='',
)
sys.stdout.flush()
if progress == 1:
conn_out.close()
print()
break
def run_daemon(qa_pipe: Connection):
wiki_daemon = WikiDaemon(WIKI_PAGE)
wiki_daemon.update_wiki_cache()
wiki_daemon.reload_spacy_docs()
print("wiki_daemon: Child process started")
next_wiki_update = time.time() + UPDATE_PERIOD_SECS
while True:
now = time.time()
if next_wiki_update < now:
print("wiki_daemon: Checking Wikipedia for updates")
updated = wiki_daemon.update_wiki_cache()
if updated:
print("wiki_daemon: Got new revision, updating")
next_wiki_update = now + UPDATE_PERIOD_SECS
if qa_pipe.poll(timeout=next_wiki_update - now):
try:
question = qa_pipe.recv()
qa_pipe.send(wiki_daemon.inquiry(question))
except EOFError:
# Pipe was closed on other end, we're done here
qa_pipe.close()
return
except ValueError:
print("Answer was too large to send!")
# Make sure the caller isn't still waiting for an object
try:
qa_pipe.send("")
except EOFError:
qa_pipe.close()
return
def process_loop(
conn: Connection, user_init: Callable[[], None], loglevel, task, task_args
):
"""Initialize a process pool worker"""
# Install SIGBUS handler (so our parent process can abort somewhat gracefully)
with suppress(AttributeError): # Windows and Cygwin do not have SIGBUS
# Windows and Cygwin do not have pthread_sigmask or SIGBUS
signal.signal(signal.SIGBUS, process_sigbus)
# Reconfigure the root logger for this process to send all messages to a queue
h = ConnectionLogHandler(conn)
root = logging.getLogger()
remove_all_log_handlers(root)
root.setLevel(loglevel)
root.addHandler(h)
user_init()
for args in task_args:
try:
result = task(args)
except Exception as e:
conn.send((MessageType.exception, e))
break
else:
conn.send((MessageType.result, result))
conn.send((MessageType.complete, None))
conn.close()
return
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters):
return False
if cls.ASYNCHRONOUS:
cls.enter_async_send_mode(send_config.get_data_to_send)
else:
cls.prepare_sync_send(ctrl_connection)
exit_requested = False
buffer_size = cls.CONTINUOUS_SEND_BUFFER_SIZE if send_config.continuous else cls.SEND_BUFFER_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
time.sleep(0.5)
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(cls.__class__.__name__))
cls.shutdown_device(ctrl_connection)
ctrl_connection.close()
class RawSocketHandler(socketserver.BaseRequestHandler):
def setup(self):
registry.append(self)
def handle(self):
self.conn = Connection(self.request.detach())
while self.conn._handle:
try:
obj = json.loads(self.conn.recv_bytes().decode('utf-8'))
except EOFError:
break
if obj['type'] == 'message':
bus.post(nt_from_dict(Message, obj['message'], None))
self.conn.send_bytes(json.dumps({'ret': True}).encode('utf-8'))
elif obj['type'] == 'request':
m = bus.post_sync(nt_from_dict(Message, obj['message'], None))
if m:
ret = {"ret": True, "response": m._asdict()}
else:
ret = {"ret": False, "response": None}
self.conn.send_bytes(json.dumps(ret).encode('utf-8'))
def send(self, msg):
if isinstance(msg, Message):
ret = {"type": "message", "message": msg._asdict()}
else:
ret = {"type": "response", "response": msg._asdict()}
self.conn.send_bytes(json.dumps(ret).encode('utf-8'))
def finish(self):
registry.remove(self)
def close(self):
self.conn.close()
def process_taxonomy_deletion(id: int, database_url: str, stop_flag: Event,
queue: Queue, log_connection: Connection):
"""
Processes taxonomy merges from a queue.
Parameters
----------
id : int
Just a id for identifying the process in the logs
database_url : str
Database URL, e.g. postgres://username:password@host:port/database
stop_flag : Event
Multiprocessing event to stop the processes
queue : Queue
Multiprocessing queue for queuing the merges
log_connection : Connection
Multiprocessing connection to send logs to the logger
"""
log_connection.send(f"deletion worker {id} is online")
database_connection = None
while not stop_flag.is_set() or not queue.empty():
if not database_connection or (database_connection and
database_connection.closed != 0):
database_connection = psycopg2.connect(database_url)
try:
# Take a taxonomy id from queue
taxonomy_id = queue.get(True, 2)
with database_connection:
with database_connection.cursor() as database_cursor:
commit_errors = 0
# Try to insert it into the database
while True:
try:
database_cursor.execute(
"DELETE FROM taxonomies WHERE id = %s;",
(taxonomy_id, ))
database_cursor.execute(
"DELETE FROM taxonomy_merges WHERE source_id = %s OR target_id = %s;",
(taxonomy_id, taxonomy_id))
database_connection.commit()
break
except BaseException as error:
database_connection.rollback()
commit_errors += 1
if commit_errors < 3:
time.sleep(random.randint(2, 5))
else:
log_connection.send(
f" taxonomy deletion {taxonomy_id} raises error:\n{error}\n"
)
break
except QueueEmptyError:
# Catch queue empty error (thrown on timeout)
continue
if database_connection and database_connection.closed == 0:
database_connection.close()
log_connection.send(f"deletion worker {id} is stopping")
# Close connection to logger
log_connection.close()
def close(data, conn: Connection):
connections.remove(conn)
for topic in topics.keys():
if conn in topics[topic]:
topics[topic].remove(conn)
if not conn.closed:
conn.close()
def Pconsumer_fcn(Pconn: Connection, loopCondition: Value):
counter = 0
while loopCondition.value == 1:
if Pconn.poll(0.1):
_ = Pconn.recv()
counter += 1
Pconn.close()
print(f"Consumer msg: Received: {counter} messages\n")
def _worker(
parent: connection.Connection,
p: connection.Connection,
env_fn_wrapper: CloudpickleWrapper,
obs_bufs: Optional[Union[dict, tuple, ShArray]] = None,
) -> None:
def _encode_obs(
obs: Union[dict, tuple, np.ndarray], buffer: Union[dict, tuple, ShArray]
) -> None:
if isinstance(obs, np.ndarray) and isinstance(buffer, ShArray):
buffer.save(obs)
elif isinstance(obs, tuple) and isinstance(buffer, tuple):
for o, b in zip(obs, buffer):
_encode_obs(o, b)
elif isinstance(obs, dict) and isinstance(buffer, dict):
for k in obs.keys():
_encode_obs(obs[k], buffer[k])
return None
parent.close()
env = env_fn_wrapper.data()
try:
while True:
try:
cmd, data = p.recv()
except EOFError: # the pipe has been closed
p.close()
break
if cmd == "step":
if data is None: # reset
obs = env.reset()
else:
obs, reward, done, info = env.step(data)
if obs_bufs is not None:
_encode_obs(obs, obs_bufs)
obs = None
if data is None:
p.send(obs)
else:
p.send((obs, reward, done, info))
elif cmd == "close":
p.send(env.close())
p.close()
break
elif cmd == "render":
p.send(env.render(**data) if hasattr(env, "render") else None)
elif cmd == "seed":
p.send(env.seed(data) if hasattr(env, "seed") else None)
elif cmd == "getattr":
p.send(getattr(env, data) if hasattr(env, data) else None)
elif cmd == "setattr":
setattr(env, data["key"], data["value"])
else:
p.close()
raise NotImplementedError
except KeyboardInterrupt:
p.close()
def log_data(conn: Connection, cs_logger):
print('Starting log...')
while True:
data = conn.recv()
if not data:
conn.close()
print('Closing...')
return
cs_logger.info(data)
def slave_scheduling(context: Context, connection: Connection):
with ProximalSimulationSlave(connection) as proxy_slave, simdag.Simulation(
context.env_file, context.task_file) as simulation:
with proxy_slave.scheduling_scope():
scheduler = SlaveScheduler(simulation, proxy_slave,
context.feature)
scheduler.run()
if context.slave_callback:
proxy_slave.send(context.slave_callback(simulation))
connection.close()
def environment_worker(
remote: Connection,
parent_remote: Connection,
env_fn_wrapper: callable,
auto_reset_on_terminal: bool = False,
render_obs_size_tuple: Optional[Tuple[int, int]] = None, # (128, 128)
):
"""
:param remote:
:param parent_remote:
:param env_fn_wrapper:
:param auto_reset_on_terminal:
:param render_obs_size_tuple:
:return:"""
warnings.simplefilter("ignore")
# with IgnoreInterruptSignal(): # TODO: DOES NOT WORK AS intended here, endless looping, needs another way to send a close signal
with suppress(UserWarning, KeyboardInterrupt):
parent_remote.close()
env = env_fn_wrapper.x()
terminated = False
while True:
cmd, data = remote.recv()
if cmd is EWC.step:
observation, signal, terminal, info = env.step(data)
if terminated:
signal = 0
if terminal:
terminated = True
if auto_reset_on_terminal:
observation = env.reset()
terminated = False
remote.send(GymTuple(observation, signal, terminal, info))
elif cmd is EWC.reset:
observation = env.reset()
terminated = False
remote.send(observation)
elif cmd is EWC.close:
env.close()
# remote.send(None)
break
elif cmd is EWC.get_spaces:
remote.send((env.observation_space, env.action_space))
elif cmd is EWC.render:
res = env.render(data)
if data != RenderModeEnum.human.value:
if render_obs_size_tuple:
res = resize(res,
render_obs_size_tuple) # VERY SLOW!!!
remote.send(res)
elif cmd is EWC.seed:
env.seed(data)
else:
raise NotImplementedError
def pyval(expr: str, pipe: Connection) -> None:
"""
Évalue une expression.
Retour (sérialisé) via pipe
"""
try:
evaluation = safe_eval(expr)
except BaseException as ex:
evaluation = ex
pipe.send(evaluation)
pipe.close()
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig,
dev_parameters: OrderedDict):
if not cls.init_device(
ctrl_connection, is_tx=True, parameters=dev_parameters):
ctrl_connection.send("failed to start tx mode")
return False
if cls.ASYNCHRONOUS:
ret = cls.enter_async_send_mode(send_config.get_data_to_send)
else:
ret = cls.prepare_sync_send(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start tx mode")
return False
exit_requested = False
buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
ctrl_connection.send("successfully started tx mode")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
try:
time.sleep(0.5)
except KeyboardInterrupt:
pass
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(),
ctrl_connection,
is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if not cls.ASYNCHRONOUS:
# Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure
# that the send buffer on the SDR is cleared
time.sleep(0.75)
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(
cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(
cls.__class__.__name__))
cls.shutdown_device(ctrl_connection, is_tx=True)
ctrl_connection.close()
def _regym_worker_shared_memory(index: int, env_fn: Callable[[], gym.Env],
pipe: Connection, parent_pipe: Connection,
shared_memory: Tuple[SynchronizedArray],
error_queue: Queue):
'''
Based on function `gym.vector.async_vector_env._worker_shared_memory`
See that function's documentation
Custom additions:
- 'environment' command: To return underlying environment
- 'step' command returns:
Note: succ_obs dimensions:
[num_agents, num_environments, environment_observations]
'''
assert shared_memory is not None
env = env_fn()
observation_space = env.observation_space
parent_pipe.close()
try:
while True:
command, data = pipe.recv()
if command == 'reset':
observation = env.reset()
write_to_shared_memory(index, observation, shared_memory,
observation_space)
pipe.send((None, True))
elif command == 'step':
observation, reward, done, info = env.step(data)
if done:
observation = env.reset()
write_to_shared_memory(index, observation, shared_memory,
observation_space)
pipe.send(((None, reward, done, info), True))
elif command == 'seed':
env.seed(data)
pipe.send((None, True))
elif command == 'close':
pipe.send((None, True))
break
elif command == 'environment':
pipe.send((env, True))
elif command == '_check_observation_space':
pipe.send((data == observation_space, True))
else:
raise RuntimeError(
'Received unknown command `{0}`. Must '
'be one of {`reset`, `step`, `seed`, `close`, `env`'
'`_check_observation_space`}.'.format(command))
except (KeyboardInterrupt, Exception):
error_queue.put((index, ) + sys.exc_info()[:2])
pipe.send((None, False))
finally:
env.close()
def receive(conn: Connection, _pipeline: Callable[[Iterator[dict]], Iterator]):
"""
Процедура получения строки через сетевое соединение и обратной передачи данных нормализации
"""
sentence = ''
stemming.init_cache()
tokenization.init_cache()
config.init_cache()
try:
with stemming.jstem_ctx() as stemmer:
while True:
if conn.poll(timeout=_RTN_CONNECTION_LIFE_TIME):
sentence = conn.recv()
logger.debug(f'Received: "{sentence}"')
if _PROFILER:
profiler.enable()
start = process_time()
try:
analisys = normalization.analyze(sentence, stemmer)
result = list(_pipeline(analisys))
except Exception as e:
logger.error(f'Normalization failed with error: {e} \n {sentence}')
result = []
end = process_time()
if _PROFILER:
profiler.disable()
conn.send(result)
logger.debug(f'Sent: {result}')
perflog.debug(f'RTN time: {round((end-start) * 1000, 2)} ms')
if _PROFILER:
profiler.dump_stats(os.path.join(settings.ROOT_PATH, f'rtn_{time()}.cprof'))
else:
raise TimeoutError
except EOFError:
logger.info('Incoming connection closed')
except TimeoutError:
logger.warning('Incoming connection timeout')
except Exception:
logger.exception(sentence if sentence else 'Normalization Failed')
finally:
stemming.cache_clear()
tokenization.cache_clear()
config.cache_clear()
logger.debug('Closing connection...')
conn.close()
logger.debug('Connection closed')
def unprocessible_proteins_worker(process_connections: list, unprocessible_proteins_fasta_path: pathlib.Path, log_connection: Connection):
log_connection.send("unprocessible proteins logger is online")
with unprocessible_proteins_fasta_path.open("w") as fasta_file:
while process_connections:
for conn in wait(process_connections):
try:
fasta_entry = conn.recv()
except EOFError:
process_connections.remove(conn)
else:
fasta_file.write(fasta_entry + "\n")
fasta_file.flush()
log_connection.send("unprocessible proteins logger is stopping")
log_connection.close()
def run(conn: Connection) -> None:
"""
Static Process Function.
When a new Backend process is started, it enters here.
Currently, it creates a WorkQueue with a worker thread
and communicates with the BQSKit Frontend via conn.
Args:
conn (Connection): The connection object used to communicate
to the frontend. Reads commands from the connection,
processes them, and responds in a loop.
"""
wq = WorkQueue()
while True:
msg = conn.recv()
if msg == 'CLOSE':
wq.stop()
conn.close()
break
elif msg == 'SUBMIT':
task = conn.recv()
if not isinstance(task, CompilationTask):
pass # TODO: Handle Error
wq.enqueue(task)
conn.send('OKAY')
elif msg == 'STATUS':
task_id = conn.recv()
if not isinstance(task_id, uuid.UUID):
pass # TODO: Handle Error
conn.send(wq.status(task_id))
elif msg == 'RESULT':
task_id = conn.recv()
if not isinstance(task_id, uuid.UUID):
pass # TODO: Handle Error
conn.send(wq.result(task_id))
elif msg == 'REMOVE':
task_id = conn.recv()
if not isinstance(task_id, uuid.UUID):
pass # TODO: Handle Error
wq.remove(task_id)
conn.send('OKAY')
def device_send(cls, ctrl_connection: Connection, send_config: SendConfig, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=True, parameters=dev_parameters):
ctrl_connection.send("failed to start tx mode")
return False
if cls.ASYNCHRONOUS:
ret = cls.enter_async_send_mode(send_config.get_data_to_send)
else:
ret = cls.prepare_sync_send(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start tx mode")
return False
exit_requested = False
buffer_size = cls.CONTINUOUS_TX_CHUNK_SIZE if send_config.continuous else cls.SYNC_TX_CHUNK_SIZE
if not cls.ASYNCHRONOUS and buffer_size == 0:
logger.warning("Send buffer size is zero!")
ctrl_connection.send("successfully started tx mode")
while not exit_requested and not send_config.sending_is_finished():
if cls.ASYNCHRONOUS:
try:
time.sleep(0.5)
except KeyboardInterrupt:
pass
else:
cls.send_sync(send_config.get_data_to_send(buffer_size))
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=True)
if result == cls.Command.STOP.name:
exit_requested = True
break
if not cls.ASYNCHRONOUS:
# Some Sync send calls (e.g. USRP) are not blocking, so we wait a bit here to ensure
# that the send buffer on the SDR is cleared
time.sleep(0.75)
if exit_requested:
logger.debug("{}: exit requested. Stopping sending".format(cls.__class__.__name__))
if send_config.sending_is_finished():
logger.debug("{}: sending is finished.".format(cls.__class__.__name__))
cls.shutdown_device(ctrl_connection, is_tx=True)
ctrl_connection.close()
def log_timer(receiver: Connection, sender: Connection):
print('downloading ', end='', flush=True)
sender.close()
# start ticker
self.ticker.start()
while True:
if receiver.poll():
done = receiver.recv()
if done:
print(flush=True)
print('download done', flush=True)
# stop ticker when download process done
self.ticker.cancel()
break
def runner(env: Env,
conn: Connection,
reset_args: list = (),
reset_kwargs: dict = {}):
while True:
action = conn.recv()
if action == EnvStatus.RESET:
state = env.reset(*reset_args, **reset_kwargs)
conn.send(state)
elif action == EnvStatus.END:
conn.close()
break
else:
state, reward, done, _ = env.step(action)
if done:
state = env.reset(*reset_args, **reset_kwargs)
conn.send((state, reward, done, _))
def env_worker(conn: Connection, **env_kwargs):
gameplay = Ccreate_start_conditions()
env = Haxball(gameplay=gameplay, **env_kwargs)
i = 0
while True:
cmd, data = conn.recv()
if cmd == 'step':
a1, a2 = data
env.step_async(a1, red_team=True)
env.step_async(a2, red_team=False)
env.step_physics()
obss = []
rews = []
dones = []
infos = []
is_done = False
for red_team in (True, False):
obs, rew, done, info = env.step_wait(red_team=red_team)
obss.append(obs)
rews.append(rew)
dones.append(done)
infos.append(info)
is_done |= done
if is_done:
env.reset()
res = np.array(obss), np.array(rews), np.array(dones), np.array(
infos)
conn.send(res)
elif cmd == 'reset':
ob = env.reset()
conn.send([ob, ob])
elif cmd == 'render':
res = env.render(mode='rgb_array')
conn.send(res)
elif cmd == 'close':
conn.close()
break
elif cmd == 'get_spaces_spec':
conn.send((env.observation_space, env.action_space, env.spec))
else:
raise NotImplementedError
def process_taxonomy_merge(id: int, database_url: str, stop_flag: Event, queue: Queue, log_connection: Connection):
log_connection.send(f"merge worker {id} is online")
# Create database connection
engine = create_engine(database_url, pool_size = 1, max_overflow = 0, pool_timeout = 3600)
SessionClass = sessionmaker(bind = engine, autoflush=False)
while not stop_flag.is_set() or not queue.empty():
# Open new session
session = SessionClass()
try:
# Take a taxonomy from queue
taxonomy_merge = queue.get(True, 2)
# Check if the merge already exists
existing_taxonomy_merge = session.query(TaxonomyMerge).filter(TaxonomyMerge.source_id == taxonomy_merge.source_id, TaxonomyMerge.target_id == taxonomy_merge.target_id).one_or_none()
if not existing_taxonomy_merge:
commit_errors = 0
# Try to insert it into the database
while True:
try:
# Insert merge
session.add(taxonomy_merge)
# Update all taxonomy id of proteins which have the old id.
session.query(Protein).filter(Protein.taxonomy_id == taxonomy_merge.source_id).update({Protein.taxonomy_id: taxonomy_merge.target_id})
session.commit()
break
except BaseException as error:
# Do a rollback
session.rollback()
commit_errors += 1
# If there are tries left, sleep between 2 and 5 second and try again
if commit_errors < 3:
time.sleep(random.randint(2, 5))
else:
# Otherwise log the error and proceed with next taxonomy
log_connection.send(f" taxonomy merge {taxonomy_merge.source_id}|{taxonomy_merge.target_id} raises error:\n{error}\n")
break
except QueueEmptyError:
# Catch queue empty error (thrown on timeout)
continue
# Close session
session.close()
# Dispose all connection within the engine
engine.dispose()
log_connection.send(f"merge worker {id} is stopping")
# Close connection to logger
log_connection.close()
def _start(self, factory: Callable[[], Env], connection: Connection):
env = factory()
_ = env.reset()
connection.send((env.observation_space, env.action_space))
terminate = False
while not terminate:
command, kwargs = connection.recv()
if command == 'render':
rendering = env.render(**kwargs)
connection.send(rendering)
elif command == 'step':
step = env.step(**kwargs)
connection.send(step)
elif command == 'reset':
obs = env.reset(**kwargs)
connection.send(obs)
elif command == 'close':
terminate = True
connection.close()
def device_receive(cls, data_connection: Connection, ctrl_connection: Connection, dev_parameters: OrderedDict):
if not cls.init_device(ctrl_connection, is_tx=False, parameters=dev_parameters):
ctrl_connection.send("failed to start rx mode")
return False
try:
cls.adapt_num_read_samples_to_sample_rate(dev_parameters[cls.Command.SET_SAMPLE_RATE.name])
except NotImplementedError:
# Many SDRs like HackRF or AirSpy do not need to calculate SYNC_RX_CHUNK_SIZE
# as default values are either fine or given by the hardware
pass
if cls.ASYNCHRONOUS:
ret = cls.enter_async_receive_mode(data_connection, ctrl_connection)
else:
ret = cls.prepare_sync_receive(ctrl_connection)
if ret != 0:
ctrl_connection.send("failed to start rx mode")
return False
exit_requested = False
ctrl_connection.send("successfully started rx mode")
while not exit_requested:
if cls.ASYNCHRONOUS:
try:
time.sleep(0.25)
except KeyboardInterrupt:
pass
else:
cls.receive_sync(data_connection)
while ctrl_connection.poll():
result = cls.process_command(ctrl_connection.recv(), ctrl_connection, is_tx=False)
if result == cls.Command.STOP.name:
exit_requested = True
break
cls.shutdown_device(ctrl_connection, is_tx=False)
data_connection.close()
ctrl_connection.close()
def statistics_worker(stop_flag: Event, statistics: Array,
write_period: int,
statistics_file_path: pathlib.Path,
log_connection: Connection):
log_connection.send("statistics logger is online")
# Snapshot of last written statistic to calculate difference
last_statistics = [0] * len(statistics)
start_at = time.time()
Digestion.print_statistic_row(Digestion.STATSTIC_FILE_HEADER)
with statistics_file_path.open("w") as statistics_file:
statistics_writer = csv.writer(statistics_file)
statistics_writer.writerow(Digestion.STATSTIC_FILE_HEADER)
statistics_file.flush()
while not stop_flag.is_set():
# Prepare array for next write
current_statistics = []
# Wait for next write
stop_flag.wait(write_period)
# Calculate seconds after start
current_time = int(time.time() - start_at)
# Get current statistics
for value in statistics:
current_statistics.append(value)
# Initialize csv row
csv_row = [current_time]
# Assign current statistics to csv row
for value in current_statistics:
csv_row.append(value)
# Calulate differences to last statistics (= rates)
for idx in range(0, len(current_statistics)):
csv_row.append(current_statistics[idx] -
last_statistics[idx])
# Write csv row to csv file
statistics_writer.writerow(csv_row)
statistics_file.flush()
# Output to console
Digestion.print_statistic_row(csv_row)
# Assign new 'snapshot'
last_statistics = current_statistics
log_connection.send("statistics logger is offline")
log_connection.close()
def _subproc(conn: Connection, remote_conn: Connection, fn_path: str) -> None:
remote_conn.close()
with open(fn_path, "rb") as f:
env = cloudpickle.load(f)()
# notify if it's ready
conn.send("ready")
while True:
command = conn.recv()
if command[0] == "step":
observation, reward, terminal, info = env.step(command[1])
conn.send([observation, reward, terminal, info])
elif command[0] == "reset":
conn.send([env.reset()])
elif command[0] == "close":
conn.close()
break
else:
raise ValueError(f"invalid {command[0]}.")
def _handle(self, conn_lru_dict: LRUCacheType[multiprocessing_connection.Connection, bool],
conn: multiprocessing_connection.Connection, c_send: multiprocessing_connection.Connection):
try:
data = conn.recv_bytes()
if not data:
raise EOFError
self.logger.debug("parse conn %s" % conn)
# self.logger.debug(data)
try:
result = self.handle(data)
if result is not None:
conn.send_bytes(result)
except Exception:
self.logger.exception("handle error")
conn_lru_dict[conn] = True
c_send.send_bytes(b'ok')
except OSError:
self.logger.debug("conn %s was closed" % conn)
conn.close()
except EOFError:
self.logger.debug("conn %s was eof" % conn)
conn.close()
except BrokenPipeError:
self.logger.debug("conn %s was broken" % conn)
conn.close()
