def play_recursive_space_cards(deck1: collections.deque,
deck2: collections.deque) -> tuple:
"""return the winner and its deck after a recursive space cards game"""
configurations = set()
while deck1 and deck2:
current_config = (tuple(deck1), tuple(deck2))
if current_config in configurations:
return 1, deck1
configurations.add(current_config)
card1, card2 = deck1.popleft(), deck2.popleft()
assert card1 != card2
if card1 <= len(deck1) and card2 <= len(deck2):
sub_deck1 = collections.deque(itertools.islice(deck1, card1))
sub_deck2 = collections.deque(itertools.islice(deck2, card2))
winner, _ = play_recursive_space_cards(sub_deck1, sub_deck2)
else:
winner = 1 if card1 > card2 else 2
if winner == 1:
deck1.extend((card1, card2))
else:
deck2.extend((card2, card1))
return (1, deck1) if deck1 else (2, deck2)
def play_cups(cups: deque, nb_moves):
for i in range(0, nb_moves):
#print(f'-- move {i+1} --')
current_cup = cups[0]
# print(cups)
# print(f'current: {current_cup}')
cups.rotate(-1)
pickups = [cups.popleft(), cups.popleft(), cups.popleft()]
destination_cup = get_destination(current_cup, cups)
# print(f'pick up: {pickups}')
# print(f'destination: {destination_cup}')
dest_in = cups.index(destination_cup)
# push dest to end right end of the queue
# and add pickups to the right
cups.rotate(-dest_in - 1)
cups.extend(pickups)
# rotate the queue back into initial state shifted once
cups.rotate(dest_in + 4)
# print('')
return cups
def bfs(stack: deque):
while stack:
current = stack.popleft()
if current:
yield current.value
stack.extend([current.left, current.right])
yield from bfs(stack)
def bfs(to_visit: deque):
while to_visit:
current = to_visit.popleft()
if current:
yield current.value
to_visit.extend([current.left, current.right])
yield from bfs(to_visit)
def recursive_combat(first_: deque, second_: deque) -> bool:
first_states = set()
second_states = set()
while len(first_) and len(second_):
if hash_deque(first_) in first_states or hash_deque(
second_) in second_states:
return True
first_states.add(hash_deque(first_))
second_states.add(hash_deque(second_))
first_card = first_.popleft()
second_card = second_.popleft()
if len(first_) >= first_card and len(second_) >= second_card:
first_won = recursive_combat(
deque(first_[x] for x in range(first_card)),
deque(second_[x] for x in range(second_card)))
else:
first_won = first_card > second_card
if first_won:
first_.extend((first_card, second_card))
else:
second_.extend((second_card, first_card))
return len(second_) == 0
def random_move_target(queue: collections.deque, opponent_board_view: Board) -> Point:
"""
Randomly choose an empty square to shoot at when in 'hunt' mode,
if hit, change to 'target' mode and focus first on neighbor cells.
"""
# decide moving mode
if len(queue) == 0:
mode = 'hunt'
else:
mode = 'target'
# make shot
if mode == 'hunt':
empty_xs, empty_ys = np.where(opponent_board_view.shots == NO_SHOT)
choice = randint(0, len(empty_xs)-1)
shot = Point(x=empty_xs[choice], y=empty_ys[choice])
elif mode == 'target':
shot = queue.popleft()
while opponent_board_view.shots[shot.x, shot.y] != NO_SHOT:
if len(queue) > 0:
shot = queue.popleft()
else:
shot = random_move_target(queue, opponent_board_view)
# add neighbors to queue if the shot is successful
if opponent_board_view.has_ship[shot.x, shot.y]:
up = Point(x=shot.x, y=max(shot.y-1, 0))
down = Point(x=shot.x, y=min(shot.y+1, 9))
left = Point(x=max(shot.x-1, 0), y=shot.y)
right = Point(x=min(shot.x+1, 9), y=shot.y)
neighbors = [up, down, left, right]
queue.extend(neighbors)
return shot
def play_part_1(deck_1: deque, deck_2: deque):
while deck_1 and deck_2:
card_1, card_2 = deck_1.popleft(), deck_2.popleft()
if card_1 > card_2:
deck_1.extend((card_1, card_2))
else:
deck_2.extend((card_2, card_1))
return deck_1 if deck_1 else deck_2
def play_space_cards(deck1: collections.deque,
deck2: collections.deque) -> collections.deque:
"""return the winner's deck after a space cards game"""
assert len(deck1) == len(deck2)
while deck1 and deck2:
card1, card2 = deck1.popleft(), deck2.popleft()
assert card1 != card2
if card1 > card2:
deck1.extend((card1, card2))
else:
deck2.extend((card2, card1))
return deck1 if deck1 else deck2
def breadth_first_search(startnode, goalnode):
queue = Queue()
queue.append(startnode)
nodesseen = set()
nodesseen.add(startnode)
while queue:
node = queue.popleft()
if node is goalnode:
return True
else:
queue.extend(node for node in node.successors if node not in nodesseen)
nodesseen.update(node.successors)
return False
async def comparePanelIds(current_panel_id, current_dialog_count, panel_history: deque):
logger = logging.getLogger(__name__)
logger.info("Checking {}...".format(AH_URL))
new_panel_id = await getPanelID()
new_dialog_count = await countPanelDialogs(panelIDToURL(new_panel_id))
if new_panel_id != current_panel_id:
if new_panel_id not in panel_history:
logger.info("New panel ID: {}->{}".format(current_panel_id, new_panel_id))
logger.info("New panel dialog count: {}".format(new_dialog_count))
panel_history.extend(new_panel_id)
await announceNewPanel(panelIDToURL(new_panel_id))
else:
logger.info("Repeated panel ID, not announcing: {}".format(new_panel_id))
logger.info("Panel history: {}".format(panel_history))
elif new_dialog_count != current_dialog_count:
logger.info("New dialog count: {}->{}".format(current_dialog_count, new_dialog_count))
await announceNewPanel(panelIDToURL(current_panel_id))
return new_panel_id, new_dialog_count
def breadth_first_search(startnode, goalnode):
queue = Queue()
queue.append(startnode)
nodesseen = set()
nodesseen.add(startnode)
while queue:
node = queue.popleft()
if node is goalnode:
return True
else:
queue.extend(node for node in node.successors
if node not in nodesseen)
nodesseen.update(node.successors)
return False
def bfs(self, board: List[List[str]], bfs_queue: deque, visited: set):
while len(bfs_queue) > 0:
cur_pos = bfs_queue.popleft()
visited.add(cur_pos)
if self.is_valid(board, cur_pos, visited):
board[cur_pos[0]][cur_pos[1]] = '-'
bfs_queue.extend([
n_pos for n_pos in [(
cur_pos[0] + 1,
cur_pos[1]), (cur_pos[0] - 1,
cur_pos[1]), (cur_pos[0], cur_pos[1] +
1), (cur_pos[0],
cur_pos[1] - 1)] if
self.is_valid(board, n_pos, visited) and n_pos not in visited
])
def breadth_first_search(startnode, goalnode):
"""
Input:
startnode: A digraph node
goalnode: A digraph node
Output:
Whether goalnode is reachable from startnode
"""
queue = Queue()
queue.append(startnode)
nodesseen = set()
nodesseen.add(startnode)
while queue:
node = queue.popleft()
if node is goalnode:
return True
else:
queue.extend(node for node in node.successors if node not in nodesseen)
nodesseen.update(node.successors)
return False
def play_game(player1: deque, player2: deque) -> int:
score = 0
while len(player1) != 0 and len(player2) != 0:
card1 = player1[0]
card2 = player2[0]
if card1 > card2:
player1.extend([card1, card2])
elif card2 > card1:
player2.extend([card2, card1])
player1.popleft()
player2.popleft()
if len(player1) == 0:
print('Player 2 is winner')
score = calculate_score(player2)
elif len(player2) == 0:
print('Player 1 is winner')
score = calculate_score(player1)
return score
def extend_state(state: collections.deque) -> Tuple[int, collections.deque]:
found = False
ind = 0
offset = 0
for i in range(5):
if state[i] == "#":
found = True
ind = i
break
if found:
offset = 5-ind
state.extendleft("." for _ in range(offset))
found = False
ind = 0
sl = len(state)
for i in range(sl - 1, sl - 6, -1):
if state[i] == "#":
found = True
ind = i
break
if found:
state.extend("." for _ in range(5-(sl-ind-1)))
return offset, state
def play_recursive(deck_1: deque, deck_2: deque):
seen_decks = set()
while deck_1 and deck_2:
id = tuple(deck_1), tuple(deck_2)
if id in seen_decks:
return True, deck_1
seen_decks.add(id)
card_1, card_2 = deck_1.popleft(), deck_2.popleft()
if len(deck_1) >= card_1 and len(deck_2) >= card_2:
sub_1, sub_2 = deque(tuple(deck_1)[:card_1]), deque(
tuple(deck_2)[:card_2])
you_did_win, _ = play_recursive(sub_1, sub_2)
else:
you_did_win = card_1 > card_2
if you_did_win:
deck_1.extend((card_1, card_2))
else:
deck_2.extend((card_2, card_1))
return (True, deck_1) if deck_1 else (False, deck_2)
def combat(p1_deck: deque, p2_deck: deque, rec=False):
game_history = set()
while p1_deck and p2_deck:
if rec:
game = tuple(p1_deck) + (0, ) + tuple(p2_deck)
if game in game_history:
return 1, p1_deck
game_history.add(game)
p1_card, p2_card = p1_deck.popleft(), p2_deck.popleft()
winner = 1 if p1_card > p2_card else 2
if rec and len(p1_deck) >= p1_card and len(p2_deck) >= p2_card:
winner, _ = combat(deque(list(p1_deck)[:p1_card]),
deque(list(p2_deck)[:p2_card]), True)
if winner == 1:
p1_deck.extend((p1_card, p2_card))
else:
p2_deck.extend((p2_card, p1_card))
return (1, p1_deck) if p1_deck else (2, p2_deck)
def play(c1: deque, c2: deque):
seen = set()
while len(c1) > 0 and len(c2) > 0:
p = pack(c1)
if p in seen:
return 0, c1
seen.add(p)
x = c1.popleft()
y = c2.popleft()
if x <= len(c1) and y <= len(c2):
w, _ = play(deque([c1[i] for i in range(x)], maxlen=51),
deque([c2[i] for i in range(y)], maxlen=51))
else:
w = x < y
if w:
c2.extend([y, x])
else:
c1.extend([x, y])
if len(c1) == 0:
return 1, c2
else:
return 0, c1
def play_recursive_combat2(player1: deque, player2: deque):
seen = set()
score = 0
while True:
p1_hash = ''.join(map(str, player1))
p2_hash = ''.join(map(str, player2))
uid = p1_hash + '0' + p2_hash
if uid in seen:
winner = 1
break
else:
seen.add(uid)
c1 = player1.popleft()
c2 = player2.popleft()
if len(player1) >= c1 and len(player2) >= c2:
sub_winner, _ = play_recursive_combat2(
deque([player1[x] for x in range(c1)]),
deque([player2[x] for x in range(c2)]))
if sub_winner == 1:
player1.extend([c1, c2])
else:
player2.extend([c2, c1])
else:
if c1 > c2:
player1.extend([c1, c2])
else:
player2.extend([c2, c1])
if len(player1) == 0:
winner = 2
score = calculate_score(player2)
break
elif len(player2) == 0:
winner = 1
score = calculate_score(player1)
break
return winner, score
def add_nums_to_deque(source_deque: deque, max_ext: int) -> None:
if max_ext < 0:
raise ValueError(f'Invalid max size ({max_ext}) for deque')
return source_deque.extend([randint(0, max_ext) for _ in range(max_ext)])
class Prime(TH.Thread):
THL = TH.Lock()
AllThreads = OD([])
__StopSignal = False
EarlyStart = True
EarlyLimit = 4
def __init__(self, Prime=0, RunID=0, JL=[], PN=''):
if (Prime < 1) or (RunID < 0) or (RunID > Prime):
raise InvalidPrimeThread # "neither ID nor JL is correct"
TH.Thread.__init__(self)
#
# self.THL.acquire()
# self.THL.release()
if True:
self.__PR = Prime # current prrime
self.__ID = RunID # 0 : dispatcher --- 1..p : worker threads
#
tn = self.__MakeName(self.__PR, self.__ID)
self.setName(tn); # easy identifier over all threads
self.AllThreads.setdefault(tn, [self, self.is_alive(), DT.datetime.now(), None, None, 0])
#
if RunID == 0:
self.__dist = product_Primes(PRIMES) # distances of start points
self.__size = length_Jumps(PRIMES) # length of input JumpList
self.__Jumpers = DQ([]) # alternativ. bytearray
PRIMES.append(self.__PR) # set the next step of primes
self.__maxx = product_Primes(PRIMES) # frame inside
self.__outL = length_Jumps(PRIMES) # assumed length of target list
#
self.__Strikes = set([]) # numbers to be removed
# self.__Strikes = set(ListOf_Strikes(JL)) # numbers to be removed
self.__OutList = DQ([]) # alternativ. bytearray
self.AddJumpList(JL) # get the first jumps and include create strikes
self.__StartCount = 0 # number of already started subthreads
self.__FinitCount = 0 # number of already ended subthreads
if RunID >= 1:
self.__parent = self.AllThreads[PN][0]
#
self.__dist = self.__parent.__dist # distances of start points
self.__size = self.__parent.__size # length of input JumpList
self.__maxx = self.__parent.__maxx # frame inside
self.__Jumpers = DQ([]) # alternativ. bytearray
#
self.__finit = 1 + (self.__ID * self.__dist)
self.__begin = self.__finit - self.__dist
print(self)
def __str__(self):
print()
print(self.getName())
print('prd:', f'{self.__maxx:>16}', ' | cnt:', f'{self.__size:>16}', sep='', end='')
if self.__ID == 0: print(' | out:', f'{self.__outL:>16}', sep='')
else: print()
print(self.__Jumpers)
if self.__ID == 0: print(self.__Strikes)
if self.__ID >= 1: print(self.__parent.getName(), self.__begin, self.__finit)
# return self.getName()
return ""
def __MakeName(self, Prime=0, RunID=0):
return 'X_' + '{:02}'.format(Prime) + '_' + '{:02}'.format(RunID)
def __MakeStrikes(self):
if (self.__ID != 0) or (len(self.__Jumpers) == 0):
return
if len(self.__Jumpers) < min(self.EarlyLimit, self.__size):
return
# print('MS:', self.__Jumpers)
self.THL.acquire()
self.__Strikes = set(ListOf_Strikes(self.__Jumpers))
self.THL.release()
def __InitWorkers(self):
if (self.__ID != 0):
return
for i in range(self.__PR):
Prime(self.__PR, i+1, PN=self.getName())
if i > 0: continue
if self.EarlyStart and (i+1 == 1):
self.__StartWorker(self.__PR, i+1)
print('ExtraStart')
def __StartWorker(self, PR=0, ID=0):
tn = self.__MakeName(PR, ID)
print('try to start: ', tn)
T = None
self.__StartCount += 1
try:
T = self.AllThreads[tn][0]
print(tn, T)
print(tn, 'alive1', T.is_alive())
T.start()
# print(tn, 'ident:', T.T.ident)
print(tn, 'alive2', T.is_alive())
delay(2000)
print(tn, 'alive3', T.is_alive())
# if T.ident() == 0:
# if T.ident() > 0:
T.start()
print(tn, 'alive4', T.is_alive())
except:
print('oops', tn)
pass
def AddJumpList(self, JL=[]):
if (self.__ID != 0):
return
if len(self.__Jumpers) + len(JL) > self.__size:
return
self.THL.acquire()
self.__Jumpers.extend(JL)
self.THL.release()
# print('JL:', self.__Jumpers)
if len(self.__Jumpers) >= min(self.EarlyLimit, self.__size):
self.__MakeStrikes() # input jumpers complete we can create all strikers
def EmergencyStop():
Prime.__StopSignal = True
delay(1500)
Prime.THL.acquire()
Z = [T for T in TH.enumerate() if T.getName().find('X_') == 0]
Prime.THL.release()
if len(Z) > 0:
print('>>> stopping threads: ', end='')
while len(Z) > 0:
T = Z.pop()
print('\t', T.getName(), end='')
# T._stop()
else:
print('\t <<<')
def run(self):
X = self.AllThreads[self.getName()]
X[3] = DT.datetime.now()
#
print('RUN', end=': ')
if self.__ID == 0:
print('dispatch', self.__PR, self.__ID)
if len(self.__Jumpers) > min(self.EarlyLimit, self.__size):
self.__InitWorkers() # create all sub threads to compute next jump list in partitions
while self.__StartCount < self.__PR:
if len(self.__Jumpers) >= self.__size:
for i in range(self.__PR):
self.__StartWorker(self.__PR, ID=i+1)
delay(1000)
elif self.__ID <= self.__PR:
print('compute', self.__PR, self.__ID)
delay(1000)
else:
print('failure', self.__PR, self.__ID)
#
X[4] = DT.datetime.now()
class PrimeDispatch(PrimeBase):
"""
holds the current prime and some common values for all worker child threads
collects after running childs the new jumper list
"""
#
def __init__(self, Prime=0, RunID=0, JL=[]):
if (Prime < 1) or (Prime > MAXPRIME) or (RunID != 0):
raise InvalidPrimeThread # out of range
PrimeBase.__init__(self, Prime, RunID)
PrimeBase.THL.acquire()
if True: # only for a good block visibility
# PRIMES has still in values
self._dist = product_Primes(PRIMES) # distances of start points
self._size = length_Jumps(PRIMES) # length of input JumpList
self._Jumpers = DQ([]) # alternatively use a bytearray
self._Strikes = set([]) # numbers to be removed
PRIMES.append(Prime) # set the next step of primes
print(PRIMES, flush=True)
# PRIMES changed for out values
self.__maxx = product_Primes(PRIMES) # size of frame inside
self.__outL = length_Jumps(PRIMES) # assumed length of target list
self._OutList = OD([])
# self.__OutList = DQ([]) # alternatively use a bytearray
#
PrimeBase.THL.release()
if len(JL) > 0:
self.AddJumpList(
JL) # get the first jumps and include create strikes
#
def __str__(self):
t = PrimeBase.__str__(self)
print(t, flush=True)
print(self._Jumpers, flush=True)
print(self._Strikes, flush=True)
return t
#
def __MakeStrikes(self):
if (len(self._Jumpers) == 0):
return
if len(self._Jumpers) < min(PrimeBase.EarlyLimit, self._size):
return
# print('MS:', self._Jumpers, flush=True)
self.THL.acquire()
self._Strikes = set(ListOf_Strikes(self._Jumpers))
self.THL.release()
#
def AddJumpList(self, JL=[]):
if len(JL) == 0: return
if len(self._Jumpers) + len(JL) > self._size:
return
if True:
self.THL.acquire()
self._Jumpers.extend(JL)
self.THL.release()
if len(self._Jumpers) >= self._size:
self.__MakeStrikes(
) # input jumpers complete we can create all strikers
if len(self._Jumpers) <= 48:
print('JL:', self._PR, self._ID, list(self._Jumpers), flush=True)
else:
print('JL:',
self._PR,
self._ID,
list(self._Jumpers)[:21],
'...',
list(self._Jumpers)[-21:],
flush=True)
#
def __InitWorkers(self):
for i in range(self._PR):
if PrimeBase._StopSignal: break
tn = self._MakeName(self._PR, i + 1)
self._OutList.setdefault(tn, [])
PrimeWorker(self._PR, i + 1, self)
#
def __StartWorker(self, PR=0, ID=0):
tn = self._MakeName(PR, ID)
try:
X = PrimeBase.AllThreads[tn]
T = X.TObject
if T.ident is None: T.start()
except:
print('problems starting thread: ', tn, flush=True)
#
def run(self):
X = self.AllThreads[self.getName()]
X.StartTime = DT.datetime.now()
print('Thread started:', self._PR, flush=True)
#
while len(self._Jumpers) < min(PrimeBase.EarlyLimit, self._size):
if PrimeBase._StopSignal: break
print(self.getName(),
'wait for jumpers before init childs',
flush=True)
delay(self._PR)
X.PauseCnt += 1
self.__InitWorkers(
) # create all sub threads to compute next jump list in partitions
delay(10 * self._PR)
while len(self._Jumpers) < self._size:
if PrimeBase._StopSignal: break
print(self.getName(),
'wait for jumpers after init childs',
flush=True)
delay(self._PR)
X.PauseCnt += 1
for i in range(self._PR):
if PrimeBase._StopSignal: break
# print(self.getName(), 'start workers b', self._PR, i+1, flush=True)
self.__StartWorker(self._PR, ID=i + 1)
delay(3)
pNextJumps = 1
pN = None
np = 0
zBreak = False
# now let it run and wait for results
while not zBreak:
if PrimeBase._StopSignal: break
zBreak = True
PrimeBase.THL.acquire()
print('get thread list', flush=True)
ZZ = [
Z for Z in PrimeBase.AllThreads.values()
if Z.Prime == self._PR and Z.RunID > 0
]
PrimeBase.THL.release()
for Z in ZZ:
zBreak &= Z.I_AmReady
print('$$: ',
Z.Prime,
Z.RunID,
Z.I_AmReady,
Z.WhereItIs,
flush=True)
# Korrektur für kleine Primzahlen, kurze Jumplisten zusammenziehen
if zBreak and self._size < PrimeBase.EarlyLimit:
tn1 = self._MakeName(self._PR, 1)
for i in range(2, self._PR + 1):
tni = self._MakeName(self._PR, i)
self._OutList[tn1].extend(self._OutList[tni])
self._OutList[tni].clear()
# Startvorbereitungen für nächste Primzahl vorbereiten
tn = self._MakeName(self._PR, 1)
if len(self._OutList[tn]) > 0: np = nextPrime(self._OutList[tn])
if (pN is None) and (1 < np <= MAXPRIME):
print('create new thread: ', np, flush=True)
pN = PrimeDispatch(np, 0)
if pN:
for Z in ZZ:
if (Z.RunID == pNextJumps) and Z.I_AmReady:
tn = self._MakeName(Z.Prime, Z.RunID)
print('working ...',
self._PR,
pNextJumps,
'-->',
np,
flush=True)
pN.AddJumpList(self._OutList[tn])
pNextJumps += 1
print('just before start ...done:',
self._PR,
pNextJumps - 1,
'-->',
np,
flush=True)
if (pN is not None) and (pNextJumps > self._PR):
print('start new thread: ', np, flush=True)
pN.start()
while True:
if PrimeBase._StopSignal: break
zi = 0
za = 0
zz = 0
for Z in PrimeBase.AllThreads.values():
if Z.Prime == self._PR and Z.RunID > 0:
if Z.InitTime is not None: zi += 1
if Z.StartTime is not None: za += 1
if Z.ReadyTime is not None: zz += 1
if zi == 0: break
if za == zz: break
delay(10 * self._PR)
X.PauseCnt += 1
#
X.FinitTime = DT.datetime.now()
print(self.getName(), 'run finished', flush=True)
def main(args):
env = GridWorld.load(args.env)
dev = torch.device(
"cuda" if args.use_cuda and torch.cuda.is_available() else "cpu")
q_net = build_MLP(2, *args.hidden_dims, 4)
target_net = build_MLP(2, *args.hidden_dims, 4)
target_net.load_state_dict(q_net.state_dict())
q_net.to(dev)
target_net.to(dev)
optim = torch.optim.SGD(q_net.parameters(), lr=args.base_lr)
if args.lr_decay is not None:
lr_sched = torch.optim.lr_scheduler.StepLR(optim, args.lr_step,
args.lr_decay)
epsilon = args.base_epsilon
memory = ReplayMemory(maxlen=args.mem_size)
avg_cumul = None
avg_success = None
avg_loss = None
AVG_R = 0.05
stats = []
try:
with tqdm.trange(args.max_iter) as progress:
for it in progress:
trajectory, cumul, success = sample_trajectory(
env, lambda z: epsilon_greedy(z, q_net, epsilon),
args.max_t)
memory.extend(trajectory)
loss = 0
for b, batch in enumerate(
sample_batch(memory, args.batch_size, args.batch_count,
dev)):
loss += update_weights(q_net, target_net, optim,
*batch[:-1], args.discount)
if b > 0:
loss /= b
avg_cumul = cumul if avg_cumul is None else (
1 - AVG_R) * avg_cumul + AVG_R * cumul
avg_success = success if avg_success is None else (
1 - AVG_R) * avg_success + AVG_R * success
avg_loss = loss if avg_loss is None else (
1 - AVG_R) * avg_loss + AVG_R * loss
lr = optim.param_groups[0]["lr"]
progress.set_postfix(cumul=avg_cumul,
success=avg_success,
loss=avg_loss,
lr=lr,
eps=epsilon)
stats.append(
(it, avg_cumul, avg_success, avg_loss, lr, epsilon))
if it % args.freeze_period == args.freeze_period - 1:
target_net.load_state_dict(q_net.state_dict())
if args.lr_decay is not None:
lr_sched.step()
if args.eps_decay is None:
epsilon = (args.base_epsilon - args.min_epsilon) * (
1 - it / args.max_iter) + args.min_epsilon
elif it % args.eps_step == args.eps_step - 1:
epsilon = max(epsilon * args.eps_decay, args.min_epsilon)
except KeyboardInterrupt:
pass
os.makedirs(args.output_dir, exist_ok=True)
with open(os.path.join(args.output_dir, "training_args.json"), 'w') as f:
json.dump(vars(args), f, indent=4)
torch.save(q_net.state_dict(),
os.path.join(args.output_dir, "trained_mlp_{}.pkl".format(it)))
with open(os.path.join(args.output_dir, "training_stats.csv"), 'w') as f:
for it_stat in stats:
f.write(', '.join(str(s) for s in it_stat))
f.write('\n')
