def _learner_loop(
self,
shared_model: torch.nn.Module,
pipes: Sequence[mp.connection.Connection],
replay_buffer_lock: mp.synchronize.Lock,
stop_event: mp.synchronize.Event,
exception_event: mp.synchronize.Event,
n_updates: Optional[int] = None,
) -> None:
try:
update_counter = 0
# To stop this loop, call stop_event.set()
while not stop_event.is_set():
# Update model if possible
if not self._can_start_replay():
continue
if n_updates is not None:
assert self.optim_t <= n_updates
if self.optim_t == n_updates:
stop_event.set()
break
if self.recurrent:
assert isinstance(self.replay_buffer,
AbstractEpisodicReplayBuffer)
with replay_buffer_lock:
episodes = self.replay_buffer.sample_episodes(
self.minibatch_size, self.episodic_update_len)
self.update_from_episodes(episodes)
else:
with replay_buffer_lock:
transitions = self.replay_buffer.sample(
self.minibatch_size)
self.update(transitions)
# Update the shared model. This can be expensive if GPU is used
# since this is a DtoH copy, so it is updated only at regular
# intervals.
update_counter += 1
if update_counter % self.actor_update_interval == 0:
with self.update_counter.get_lock():
self.update_counter.value += 1
shared_model.load_state_dict(self.model.state_dict())
# To keep the ratio of target updates to model updates,
# here we calculate back the effective current timestep
# from update_interval and number of updates so far.
effective_timestep = self.optim_t * self.update_interval
# We can safely assign self.t since in the learner
# it isn't updated by any other method
self.t = effective_timestep
if effective_timestep % self.target_update_interval == 0:
self.sync_target_network()
except Exception:
self.logger.exception("Learner loop failed. Exiting")
exception_event.set()
def run_protocol_listener(port: int, dhtid: DHTID, started: mp.synchronize.Event, ping: Optional[Endpoint] = None):
loop = asyncio.get_event_loop()
protocol = loop.run_until_complete(DHTProtocol.create(
dhtid, bucket_size=20, depth_modulo=5, num_replicas=3, wait_timeout=5, listen_on=f"{LOCALHOST}:{port}"))
assert protocol.port == port
print(f"Started peer id={protocol.node_id} port={port}", flush=True)
if ping is not None:
loop.run_until_complete(protocol.call_ping(ping))
started.set()
loop.run_until_complete(protocol.server.wait_for_termination())
print(f"Finished peer id={protocol.node_id} port={port}", flush=True)
def _poller_loop(
self,
shared_model: torch.nn.Module,
pipes: Sequence[mp.connection.Connection],
replay_buffer_lock: mp.synchronize.Lock,
stop_event: mp.synchronize.Event,
exception_event: mp.synchronize.Event,
) -> None:
# To stop this loop, call stop_event.set()
while not stop_event.is_set() and not exception_event.is_set():
time.sleep(1e-6)
# Poll actors for messages
for i, pipe in enumerate(pipes):
self._poll_pipe(i, pipe, replay_buffer_lock, exception_event)
def _process_worker(self, stream: ZeroStream,
successful_event: multiprocessing.synchronize.Event):
node = stream.pipe.node_b
if isinstance(node, str):
return
self._process_stream_start(stream)
try:
node.process(stream, wait_timeout=self.stream_waiting_timeout)
except Exception as e:
return self._process_stream_error(stream, e)
self._process_stream_result(stream)
successful_event.set()
def _poll_pipe(
self,
actor_idx: int,
pipe: mp.connection.Connection,
replay_buffer_lock: mp.synchronize.Lock,
exception_event: mp.synchronize.Event,
) -> None:
if pipe.closed:
return
try:
while pipe.poll() and not exception_event.is_set():
cmd, data = pipe.recv()
if cmd == "get_statistics":
assert data is None
with replay_buffer_lock:
stats = self.get_statistics()
pipe.send(stats)
elif cmd == "load":
self.load(data)
pipe.send(None)
elif cmd == "save":
self.save(data)
pipe.send(None)
elif cmd == "transition":
with replay_buffer_lock:
if "env_id" not in data:
data["env_id"] = actor_idx
self.replay_buffer.append(**data)
self._cumulative_steps += 1
elif cmd == "stop_episode":
idx = actor_idx if data is None else data
with replay_buffer_lock:
self.replay_buffer.stop_current_episode(env_id=idx)
stats = self.get_statistics()
pipe.send(stats)
else:
raise RuntimeError(
"Unknown command from actor: {}".format(cmd))
except EOFError:
pipe.close()
except Exception:
self.logger.exception("Poller loop failed. Exiting")
exception_event.set()
def publisher_process(shutdown_event: multiprocessing.synchronize.Event,
dispatcher_queue: EventQueue) -> None:
print('publisher starting')
try:
while not shutdown_event.is_set():
dispatcher_queue.publish(request_id=process_publisher_request_id,
event_name=eventNames.WORK_STARTED,
event_payload={'time': time.time()},
publisher_id='eventing_pubsub_process')
except KeyboardInterrupt:
pass
print('publisher shutdown')
def _worker_loop(
input_queue: multiprocessing.Queue,
output_queue: multiprocessing.Queue,
transform_stream,
stop_event: multiprocessing.synchronize.Event,
):
"""
Do the actual work.
1. Get an object from the input queue.
2. Transform this object, and
3. Put all resulting objects onto the result queue.
4. Put an additional _Signal.YIELD onto the result queue to signalize
to the reader that it may switch to another worker.
"""
try:
while True:
input_obj = _get_until_stop(input_queue, stop_event)
if input_obj is _Signal.END:
# Nothing is left to be done
output_queue.put(_Signal.END)
break
try:
# Transform object
for output_obj in transform_stream([input_obj]):
if stop_event.is_set():
# If stop event is set, quit processing this object
break
# Put results onto the output_queue
output_queue.put(output_obj)
except: # pylint: disable=bare-except
# Put exception to queue and quit processing
output_queue.put(
ExceptionWrapper("worker process PID {}".format(os.getpid()))
)
break
finally:
# Signalize the collector to switch to the next worker
output_queue.put(_Signal.YIELD)
except KeyboardInterrupt:
pass
except _Stop:
pass
def handle_new_block(
self,
q_object_from_compete_process_to_mining,
q_for_block_validator,
is_generating_block: multiprocessing.synchronize.Event,
has_received_new_block: multiprocessing.synchronize.Event,
is_not_in_process_of_creating_new_block: multiprocessing.
synchronize.Event,
is_program_running: multiprocessing.synchronize.Event,
pause_time=60):
# todo: a queue object should wait for 5 random signed bytes in a beta version, for now not needed
# todo: for now q object will wait for 7 seconds
# todo: get transactions, misc messages, wallet hash state dicts
# todo: receive any extra messages and add to appropriated dict before start time
# todo: while generating block, allow for receiving of transactions/msgs for next competition
start_time, len_of_competition, single_prime_char, exp_leading_prime, new_block_no, prev_hash = \
None, None, None, None, None, None
addl_chars = None
tx_misc_wsh = None
# None == has not generated block or Just finished generating
# False == received a recent block and is waiting for start time, during will received random signed bytes
# True means
while is_program_running.is_set():
try:
rsp = q_object_from_compete_process_to_mining.get(timeout=0.25)
print("received rsp in Handle new block")
except Empty:
if is_program_running.is_set() is False:
print("Program should end")
pass
else:
if isinstance(rsp, str) and rsp in {
'exit', 'quit'
}: # exit signal to stop program
print("Exiting From Mining Process")
break
elif isinstance(rsp, list) and len(rsp) >= 2:
if rsp[0] == "bcb": # new block! new arguments ['bcb', block, class_holding tx_misc, wsh]
has_received_new_block.set()
start_time, len_of_competition, single_prime_char, exp_leading_prime, new_block_no,\
addl_chars, prev_hash = self.get_new_block_arguments(rsp=rsp, pause_time=pause_time)
tx_misc_wsh = rsp[2]
print(f"In Handle blocks, {start_time}, {time.time()}")
elif rsp[
0] == 'm': # rsp == ['m', msg hash, misc_msg_list]
try:
tx_misc_wsh.add_to_misc_msg(msg_hash=rsp[1],
msg=rsp[2])
except AttributeError as e: # tx_misc_wsh is still None
print(f"tx_misc_wsh is still none: {e}")
elif rsp[0] == "wsh": # for wallet state hash
# todo: wallet hash state key is the wallet and a dict
# todo: this dict has keys {"w_hash": str, "start": int amount, "end": int amount, "act": set of hashes of activities found in block activity }
pass
elif rsp[
0] == "bk0": # sending genesis block, network just launched
start_time, len_of_competition, single_prime_char, exp_leading_prime, new_block_no, \
addl_chars, prev_hash = self.get_block_one_arguments()
tx_misc_wsh = rsp[2]
has_received_new_block.set()
print(f"In Handle blocks, {start_time}, {time.time()}")
else: # rsp SHOULD represent a transaction rsp == [tx_type, tx_hash, [main_msg, sig]]
tx_misc_wsh.add_to_txs(
type_of_tx=rsp[0],
tx_hash=rsp[1],
tx=rsp[2],
fees=rsp[2][0]
['fee'] # rsp[2] = [main_msg, sig] in main_msg 'fee' key
)
# todo: implement this later:
# todo: if 5 random bytes already received changed to and (time.time() >= start_time or random_received => 5)
try:
if is_generating_block.is_set(
) is False and has_received_new_block.is_set(
) is True and time.time() >= start_time:
# set this to false
# at compete it is set to true, when a new block has been received
# used by exit process to determine when to exit, it waits for process not to be creating blocks
is_not_in_process_of_creating_new_block.clear()
print(
"in handle_new_block, Orses_Compete_Algo.py, Bout To Start Competing"
)
# generating block is and instance of multiprocessing.Event()
is_generating_block.set()
# ***** this is a blocking code **** #
new_block = generate_regular_block(
exp_leading_prime=exp_leading_prime,
len_competiion=len_of_competition,
single_prime_char=single_prime_char,
wsh=tx_misc_wsh.wsh,
block_no=new_block_no,
fees=tx_misc_wsh.fees,
combined_list=tx_misc_wsh.combined_list_of_hashes,
admin_inst=self.admin_inst,
primary_sig_wallet=self.reward_wallet,
addl_chars=addl_chars,
no_of_txs=tx_misc_wsh.number_of_transactions,
no_of_asgns=tx_misc_wsh.
number_of_assignment_statements,
prev_hash=prev_hash,
is_program_running=is_program_running)
# once done clear event object (becomes false) to notify compete_process
is_generating_block.clear()
has_received_new_block.clear()
# todo: when new block is created, send block to blockchainPropagatorInitiator process
# todo: this is done by sending using validator to blockchain queue ie q_for_block_validator
print(
f"just created block: {new_block}, is generating block: {is_generating_block.is_set()}"
)
reason_msg = "nb" if new_block_no > 1 else "nb1"
# how long the time to receive blocks from other nodes should last
end_time = time.time() + (pause_time / 2)
# To account for the fact that there might not be a valid hash
# if not a valid has, then block_hash will be None
if new_block["bh"]["block_hash"]:
# this goes to block initiator method process of BlockchainPropagator
end_time = time.time() + (pause_time / 2)
q_for_block_validator.put(
[reason_msg, end_time, new_block])
else:
# todo: send list similar to non_compete process, when valid hash not found
# list of args = [block_no, prime char, addl_chars, exp_leading_prime]
# [reason_msg, end_time, new_block or list of args]
# set to true, if no hash was found, no need to wait if trying to exit.
# because there is no likely hood a monetary loss
is_not_in_process_of_creating_new_block.set()
q_for_block_validator.put([
"new_round", # reason message
time.time() + (pause_time / 2), # end time
[
new_block_no, single_prime_char, addl_chars,
exp_leading_prime
],
])
except TypeError as e:
print(f"in Orses Compete error: {e}")
continue
def compete(
self, q_for_compete: (multiprocessing.queues.Queue, queue.Queue),
q_object_from_compete_process_to_mining: (multiprocessing.queues.Queue,
queue.Queue),
is_generating_block: multiprocessing.synchronize.Event,
has_received_new_block: multiprocessing.synchronize.Event,
is_not_in_process_of_creating_new_block: multiprocessing.synchronize.
Event):
print(
f"in Orses_compete_alog, Started Compete Process For admin: {self.admin_inst.admin_name}"
)
recent_blk = q_for_compete.get()
print(
f"in Orses_compete_Algo, recent block:\n{recent_blk} admin: {self.admin_inst.admin_name}"
)
reason_dict = dict()
reason_dict['b'] = "ttx"
reason_dict['c'] = "rsv_req"
reason_dict['d'] = "rvk_req"
reason_dict['f'] = "misc_msg"
if "bh" in recent_blk and recent_blk["bh"]:
recent_block_no = int(recent_blk['bh']["block_no"], 16)
else:
return
# instantiate class with existing txs, misc_msg and wsh. this is passed to self.handle_new_block()
tx_misc_wsh = TxMiscWsh(txs=self.create_empty_tx_dict(), fees=0)
# if genesis block is most recent and network was just launched then run this
if self.just_launched and recent_block_no == 0:
print(f"in {__file__}: Just launched so should be mining")
q_object_from_compete_process_to_mining.put(
['bk0', recent_blk, tx_misc_wsh])
else:
print(f"Not mining {recent_blk}, {recent_block_no}")
msg_count = 0
while self.is_program_running.is_set():
rsp = q_for_compete.get(
) # [reason letter, main tx dict OR main block dict]
if self.is_program_running.is_set():
msg_count += 1
print(
f"in Orses_compete, msg sent, msg count is {msg_count}: {rsp}"
)
# rsp should be dictionary of transaction ie
if isinstance(rsp, str) and rsp in {"exit", "quit"}:
q_object_from_compete_process_to_mining.put(rsp)
if is_generating_block:
print(
"Currently Generating Block. Program Will End After"
)
break
elif rsp[
0] == 'bcb': # rsp is a block bcb == blockchain block, rsp = ['bcb', block]
received_block_no = int(rsp[1]['bh']["block_no"], 16)
try:
print(
f"received block no {received_block_no}, recent block +1 = {recent_block_no+1}"
)
assert received_block_no == recent_block_no + 1
except AssertionError as e:
print(
f'Assertion_error in compete(), Orses_compete_algo.py {e}'
)
break
recent_block_no = received_block_no
# add previous tx_misc_wsh
rsp.append(
tx_misc_wsh) # rsp == ['bcb', block, tx_misc_wsh]
# this is set to True, if exit process waiting for when not creating block to exit
is_not_in_process_of_creating_new_block.set()
# has_received_new_block is set to true in mining process
q_object_from_compete_process_to_mining.put(rsp)
# instantiate a new tx_misc_wsh
tx_misc_wsh = TxMiscWsh(txs=self.create_empty_tx_dict(),
fees=0)
else:
if rsp[0] == "a": # assignment statements are not included directly
print(
f"received an assignment statement in compete, SHOULD NOT RECEIVE IN COMPETE\n{rsp}"
)
pass
elif rsp[0] == "f": # misc messages
main_msg = rsp[1]["misc_msg"]
sig = rsp[1]['sig']
# if new block has been received but next block not being generated,
if has_received_new_block.is_set(
) is True and is_generating_block.is_set() is False:
# this goes to process being run self.handle_new_block Competitor method
q_object_from_compete_process_to_mining.put(
['m', rsp[1]['msg_hash'], misc_m])
else:
tx_misc_wsh.add_to_misc_msg(
msg_hash=rsp[1]["msg_hash"],
msg=[main_msg, sig],
fees=main_msg['fee'])
else: # transaction message
# todo: when checking transaction messages, check for fees
try:
# rsp[0] either b,c,d
# tx_dict_key either 'ttx', 'rsv_req' or 'rvk_req'
tx_dict_key = reason_dict.get(rsp[0], None)
if tx_dict_key:
tx_dict_key = reason_dict[rsp[0]]
main_msg = rsp[1][tx_dict_key]
sig = rsp[1]['sig']
tx_hash = rsp[1]["tx_hash"]
if has_received_new_block.is_set(
) is True and is_generating_block.is_set(
) is False:
# print("is here 111")
q_object_from_compete_process_to_mining.put(
[
tx_dict_key, tx_hash,
[main_msg, sig]
])
else:
tx_misc_wsh.add_to_txs(
type_of_tx=tx_dict_key,
tx_hash=tx_hash,
tx=[main_msg, sig],
fees=main_msg['fee'])
except KeyError as e:
print(
f"In Orses_compete_algo: compete(), Key Error: {e},\nmsg: {rsp}\n"
)
continue
print(
f"In Orses_Compete_Algo.py, this is tx_misc_wsh {vars(tx_misc_wsh)}"
)
print("in Orses_Compete_Algo: Compete, process done")
