def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=16)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
# add widgets
self.title = tk.Label(self,
text="AI In The 21st Century",
font=("Courier", 48))
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
artificial_intelligence_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
briefly_on_q_learning_and_deep_q_network_frame),
text="Next",
font=("Courier", 14))
# add + lay toggle widgets (args: self, master, text_string, width=1, height=1, wraplength=None, justify=tk.LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text1 = eh.toggle_label(
self,
"Today, we can see applications of AI in branches such as:\n- natural language processing\n- speech recognition\n- face recognition\n- computer vision\n- strategic planning and computer game bot\nFor this tutorial, we are going to focus on 'strategic planning and computer game bot'.",
row=1,
column=1,
wraplength=900)
self.text2 = eh.toggle_label(
self,
"In 2016, game artificial intelligence was under heavy highlight for the highly-anticipated match between 18-times Go champion Lee Sedol and an AI agent named AlphaGo, developed by Google's Deepmind team. This match astonished the world as AlphaGo succeeded in winning the series 4-1.",
row=2,
column=1,
wraplength=900)
self.text3 = eh.toggle_label(
self,
"This was a major milestone in AI research as Go has been traditionally considered to be too hard a problem for a machine learning algorithm. AlphaGo's machine learning algorithm employs a technique called reinforcement, a specific type of which we will cover in this tutorial.",
row=3,
column=1,
wraplength=900)
# lay widgets
self.title.grid(row=0, column=0, columnspan=8)
self.previous_frame_button.grid(row=4, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=4, column=1, columnspan=1)
self.next_frame_button.grid(row=4, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=12)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
# add widgets
self.title = tk.Label(self,
text="Briefly On Q-Learning & Deep Q-Network",
font=("Courier", 48))
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
ai_in_the_21st_century_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(play.pong_gameplay_frame
),
text="Next",
font=("Courier", 14))
# add + lay toggle widgets (args: self, master, text_string, width=1, height=1, wraplength=None, justify=tk.LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text1 = eh.toggle_label(
self,
"Q-Learning is a reinforcement learning technique that can be used to find the best policy for taking actions. The idea was first introduced by Christopher Watkins in 1989 and the convergence of its optimal policy search was proved by Watkins and Peter Dayans in 1992.",
row=1,
column=1,
wraplength=900)
self.text2 = eh.toggle_label(
self,
"Deep Q-Network is an algorithm that combines the branches of Q-Learning with neural network. This was done to combat the relatively small workspace Q-Learning algorithm can work with. With recent advances in neural network techniques, Google's Deepmind team was able to create an algorithm that performed as well as or better than humans in numerous Atari 2006 games, with breakthrough publications in 2013 and in 2015.",
row=2,
column=1,
wraplength=900)
# lay widgets
self.title.grid(row=0, column=0, columnspan=8)
self.previous_frame_button.grid(row=3, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=3, column=1, columnspan=1)
self.next_frame_button.grid(row=3, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=6)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
self.rowconfigure(7, weight=1)
self.rowconfigure(8, weight=1)
self.rowconfigure(9, weight=1)
self.rowconfigure(10, weight=1)
self.rowconfigure(11, weight=1)
# add widgets
self.title = tk.Label(self,
text="Tutorial Overview",
font=("Courier", 48))
self.text1 = tk.Label(self,
text="Total duration",
font=("Courier", 16))
self.text3 = tk.Label(
self,
text="What do I need to know before this tutorial?",
font=("Courier", 16))
self.text5 = tk.Label(
self,
text="What will I leave with at the end of this tutorial?",
font=("Courier", 16))
self.text7 = tk.Label(self,
text="How is this tutorial designed?",
font=("Courier", 16))
self.text9 = tk.Label(self,
text="How do I navigate this tutorial?",
font=("Courier", 16))
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
artificial_intelligence_frame),
text="Next",
font=("Courier", 14))
# add + lay toggle widgets (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(self,
"- Around 60 minutes",
wraplength=900,
row=2,
column=1)
self.text4 = eh.toggle_label(
self,
"- A basic understanding of what an algorithm is",
wraplength=900,
row=4,
column=1)
self.text6 = eh.toggle_label(
self,
"- An in-depth knowledge of how the basic 'Q-Learning' algorithm works\n- An in-depth knowledge of how the basic 'Deep Q-Learning' (DQN) algorithm works\n- An experiential understanding of how a DQN learner develops over time",
wraplength=900,
row=6,
column=1)
self.text8 = eh.toggle_label(
self,
"This tutorial is designed to build layer upon layer of concepts. By integrating real-time gameplay and examples from the classic arcade video game 'Pong', this tutorial will incorporate as much experiential and graphic learning as possible to cover all the mathematical and algorithmic concepts.",
width=1,
height=2,
wraplength=900,
row=8,
column=1)
self.text10 = eh.toggle_label(
self,
"You can move freely back and forth between pages by clicking on the buttons below. Any progress you make in a page will be left saved even if you leave that page.",
wraplength=900,
row=10,
column=1)
# lay widgets
self.title.grid(row=0,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.text1.grid(row=1,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.text3.grid(row=3,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.text5.grid(row=5,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.text7.grid(row=7,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.text9.grid(row=9,
column=1,
rowspan=1,
columnspan=1,
sticky=tk.NW + tk.SE)
self.previous_frame_button.grid(row=11, column=0, sticky=tk.W)
self.to_roadmap_button.grid(row=11, column=1)
self.next_frame_button.grid(row=11, column=2, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
# add widgets
self.title = tk.Label(self,
text="Artificial Intelligence",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
justify=tk.LEFT,
wraplength=1000,
text=
"So what we are about to learn today falls under a broad category that is called 'artificial intelligence'. We hear about this term all the time, artificial intelligence. But what does it mean to begin with?"
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(tutorial_overview_frame
),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
ai_in_the_21st_century_frame),
text="Next",
font=("Courier", 14))
# add + lay toggle widgets (args: self, master, text_string, width=1, height=1, wraplength=None, justify=tk.LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self,
"Well, according to the Merriam Webster dictionary, artificial intelligence is:\n1. a branch of computer science dealing with the simulation of intelligent behavior in computers.\n2. the capability of a machine to imitate intelligent human behavior.",
row=2,
column=1,
wraplength=1000)
self.text3 = eh.toggle_label(
self,
"In other words, artificial intelligence is a technique with which we equip our computer to behave like humans. As a tool that we humans have developed to help us solve our problems, artificial intelligence distinguishes itself from other types of tools in that we are creating a machine that supplements the function of the human brain and not something that merely supplements the function of a leg or an arm.",
row=3,
column=1,
wraplength=1000)
self.text4 = eh.toggle_label(
self,
"This type of cognitive function requires two things: to perceive the environment and to make appropriate decisions.",
row=4,
column=1,
wraplength=1000)
# lay widgets
self.title.grid(row=0, column=0, columnspan=8)
self.text1.grid(row=1, column=1)
self.previous_frame_button.grid(row=5, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=5, column=1, columnspan=1)
self.next_frame_button.grid(row=5, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=8)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
# add widgets
self.title = tk.Label(
self,
text="Action-Value Function Revisited: Discount Factor",
font=("Courier", 36))
self.row_one_frame = tk.Frame(self)
self.row_two_frame = tk.Frame(self)
self.row_five_frame = tk.Frame(self)
self.text1 = tk.Label(
self.row_one_frame,
font=("Courier", 14),
wraplength=500,
justify=tk.LEFT,
text="Now, let's go back to our action-value function!")
self.previous_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(
q_learning_algorithm_exploration_v_exploitation_frame),
self.controller.frames[
q_learning_algorithm_exploration_v_exploitation_frame].
video_animation.open_video()
],
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
dqn.the_large_problem_with_pixel_input_frame),
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
q_function_original_image_scale = 0.5
q_function_original_image_path = 'media/module_q/action_value_function_revisited_discount_factor_frame/original.png'
self.q_function_original_image = ImageTk.PhotoImage(
im.scale_image(Image.open(q_function_original_image_path),
q_function_original_image_scale))
self.image_one = eh.toggle_image(self.row_one_frame,
image=self.q_function_original_image,
row=0,
column=1,
width=600,
height=100)
states_image_scale = 0.5
states_image_path = 'media/module_q/action_value_function_revisited_discount_factor_frame/states.png'
self.states_image = ImageTk.PhotoImage(
im.scale_image(Image.open(states_image_path), states_image_scale))
self.image_two = eh.toggle_image(self.row_two_frame,
image=self.states_image,
row=0,
column=0,
width=600,
height=200)
self.text2 = eh.toggle_label(
self.row_two_frame,
"As you can see, the action-value for certain (action, state) pairs are all equal to one another at 10, regardless of how far away it may be in time to the actual reward-reaping instance!",
wraplength=300,
font_size=14,
row=0,
column=1,
width=40)
self.text3 = eh.toggle_label(
self,
"Indeed, with certain degrees of uncertainty as introduced with the epsilon variable from before, this can become problematic. What's more, we also have to take into account the various uncertainties derived from our adversary and the environment itself.",
wraplength=1000,
font_size=14,
row=3,
column=1,
height=2)
self.text4 = eh.toggle_label(
self,
"The idea is that future rewards that are far away should be counted less, since there is a higher uncertainty of reaching that reward-reaping state.",
wraplength=1000,
font_size=14,
row=4,
column=1,
height=2)
q_function_updated_image_scale = 0.5
q_function_updated_image_path = 'media/module_q/action_value_function_revisited_discount_factor_frame/updated.png'
self.q_function_updated_image = ImageTk.PhotoImage(
im.scale_image(Image.open(q_function_updated_image_path),
q_function_updated_image_scale))
self.image_three = eh.toggle_image(self.row_five_frame,
image=self.q_function_updated_image,
row=0,
column=1,
width=600,
height=100)
self.text5 = eh.toggle_label(
self.row_five_frame,
"And we can do this by casting a bit of doubt to the 'forward-planning' aspect of our action-value function, by multiplying with a gamma constant between 0 and 1 representing what's known as the 'discount factor.'",
wraplength=300,
font_size=14,
width=40,
row=0,
column=0)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.row_one_frame.grid(row=1, column=1)
self.row_two_frame.grid(row=2, column=1)
self.row_five_frame.grid(row=5, column=1)
self.text1.grid(row=0, column=0)
self.previous_frame_button.grid(row=6, column=0, padx=0, sticky=tk.W)
self.to_roadmap_button.grid(row=6, column=1, columnspan=1)
self.next_frame_button.grid(row=6, column=7, padx=0, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
self.rowconfigure(7, weight=1)
self.rowconfigure(8, weight=1)
self.rowconfigure(9, weight=1)
self.rowconfigure(10, weight=1)
self.rowconfigure(11, weight=1)
self.pygame_event = eh.pygame_event(self)
self.player_one_score = tk.StringVar()
self.player_two_score = tk.StringVar()
self.current_level = tk.StringVar()
# add widgets
self.title = tk.Label(self,
text="Challenge: DQN Player!",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
wraplength=1000,
text=
"Now let's play against a trained DQN player! This player was trained using the 'Pong' specifications that we have discussed thus far in this tutorial.\n\nFor this challenge, however, we are going to change up the rules a little bit, just for this gameplay:"
)
self.challenge_level_one_button = tk.Button(
self,
command=lambda: self.challenge_gameplay("one_million"),
text="Challenge: Level 1",
font=("Courier", 16))
self.challenge_level_two_button = tk.Button(
self,
command=lambda: self.challenge_gameplay("two_million"),
text="Challenge: Level 2",
font=("Courier", 16))
self.challenge_level_three_button = tk.Button(
self,
command=lambda: self.challenge_gameplay("five_million"),
text="Challenge: Level 3",
font=("Courier", 16))
self.challenge_level_four_button = tk.Button(
self,
command=lambda: self.challenge_gameplay("ten_million"),
text="Challenge: Level 4",
font=("Courier", 16))
self.challenge_level_five_button = tk.Button(
self,
command=lambda: self.challenge_gameplay("fourteen_million"),
text="Challenge: Level 5",
font=("Courier", 16))
self.scoreboard = tk.Frame(self)
self.player_one_score_label = tk.Label(
self.scoreboard,
textvariable=self.player_one_score,
font=("Courier", 16))
self.player_two_score_label = tk.Label(
self.scoreboard,
textvariable=self.player_two_score,
font=("Courier", 16))
self.current_level_label = tk.Label(self.scoreboard,
textvariable=self.current_level,
font=("Courier", 16))
self.player_one_score_label.grid(row=0, column=0)
self.player_two_score_label.grid(row=0, column=2)
self.current_level_label.grid(row=0, column=1)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(dqn.replay_memory_frame
),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
reflection_on_evolution_of_dqn_player_frame),
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self,
"- Instead of using the scoring system from previous, we will now define a point as that when a player succeeds in getting the ball past its opponent. The first player to reach the score of 3 is the winnner.",
wraplength=1000,
font_size=16,
row=2,
column=1,
width=10,
height=3)
self.text3 = eh.toggle_label(
self,
"- If you win against the DQN Player, you will move onto the next level. Upon losing or beating Level 5, the game will exit.",
wraplength=1000,
font_size=16,
row=3,
column=1,
width=10,
height=3)
self.text4 = eh.toggle_label(
self,
"Training Specifications:\nLevel One: 1,000,000 frames\nLevel Two: 2,000,000 frames\nLevel Three: 5,000,000 frames\nLevel Four: 10,000,000 frames\nLevel Five: 14,000,000 frames",
wraplength=1000,
font_size=16,
row=4,
column=1,
width=10,
height=6)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.text1.grid(row=1, column=1)
self.challenge_level_one_button.grid(row=5, column=1)
self.challenge_level_two_button.grid(row=6, column=1)
self.challenge_level_three_button.grid(row=7, column=1)
self.challenge_level_four_button.grid(row=8, column=1)
self.challenge_level_five_button.grid(row=9, column=1)
self.scoreboard.grid(row=10, column=1)
self.previous_frame_button.grid(row=11, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=11, column=1, columnspan=1)
self.next_frame_button.grid(row=11, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
# add widgets
self.title = tk.Label(self,
text="Q-Learning Algorithm: Initialization",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
wraplength=1000,
justify=tk.LEFT,
text=
"Now, let's see how our Q-Learning Agent equipped with the 'Q-Learning' algorithm actually plays 'Pong'!"
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
action_value_function_max_q_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(
q_learning_algorithm_training_sequence_frame), self.
controller.frames[q_learning_algorithm_training_sequence_frame
].video_animation.open_video()
],
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
init_image_scale = 0.5
init_image_path = 'media/module_q/q_learning_algorithm_initialization_frame/initialization.png'
self.init_image = ImageTk.PhotoImage(
im.scale_image(Image.open(init_image_path), init_image_scale))
self.text2 = eh.toggle_label(
self,
"First thing's first, our Q-Learning Agent needs to iterate through every possible (state, action) pairs and initialize their action-values to 0.",
wraplength=1000,
row=2,
column=1)
self.initialization_image = eh.toggle_image(self,
image=self.init_image,
row=3,
column=1,
width=1000,
height=200)
self.text4 = eh.toggle_label(
self,
"Then, the game will start and our computer player will observe a state in the form of pixel data. It will compare all available action-values (3 of them in our case: up, down, or staying still) at the current state, and simply choose to take the action with the largest action-value.",
wraplength=1000,
row=4,
column=1)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.text1.grid(row=1, column=1)
self.previous_frame_button.grid(row=5, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=5, column=1, columnspan=1)
self.next_frame_button.grid(row=5, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=8)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
self.epsilon_value = tk.StringVar()
# add widgets
self.title = tk.Label(
self,
text="Q-Learning Algorithm: Exploration v. Exploitation",
font=("Courier", 36))
self.text1 = tk.Label(
self,
font=("Courier", 12),
wraplength=1000,
justify=tk.LEFT,
text=
"In the beginning, choosing actions randomly was the only choice for our Q-Learning Agent, since every action-value was initialized to the identical and trivial value of 0. However, upon a series of random actions - or exploration - our Q-Learning Agent came across an instance of non-trivial action-value update. How exciting!\n\nLet's assume that our Q-Learning Agent continues with its training, and it comes across numerous non-trivial updates to the point where it no longer needs to choose actions randomly to break ties on trivial action-values. So now that our Q-Learning Agent has a non-trivial action-value system, does it make sense for it to continue with its training by only following the current method of blindly taking the action with the highest action-value?"
)
self.previous_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(
q_learning_algorithm_training_sequence_frame),
self.video_animation.close_video(), self.controller.frames[
q_learning_algorithm_training_sequence_frame].
video_animation.open_video()
],
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(prelude.roadmap_frame),
self.video_animation.close_video()
],
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(
action_value_function_revisited_discount_factor_frame),
self.video_animation.close_video()
],
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self,
"Think about this scenario: your town has a McDonald's, a Wendy's, and a Burger King. You like burgers and you frequent the local burger joint, but you have only been to McDonald's thus far. You like McDonald's. However, does this necessarily mean that you should always keep going to McDonald's from now on? What if it turns out that your taste buds are more suited for the Whoppers from Burger King after all?",
wraplength=1000,
font_size=12,
row=2,
column=1)
self.text3 = eh.toggle_label(
self,
"Similarly, what if the current action-value system does not take into account all possible ways of how best to reap the rewards? What if our Q-Learning algorithm is stuck at a local maxima, and it needs to 'go outside the box' of the current action-value system to properly consider all options?",
wraplength=1000,
font_size=12,
row=3,
column=1)
self.text4 = eh.toggle_label(
self,
"To this logic, it is useful that our Q-Learning Agent explore erratically during the beginning stages of its training, and eventually cool down the randomness for modest yet persistent exploration onwards. At the beginning, our algorithm needs to avoid focusing only on a handful instances of early reward scenarios, which most likely are not accurate representations of the overall gameplay to come. On a separate note, after enough 'exploration', the frequency of the randomness should decrease, so as to be more focused on fine-tuning the action-value system itself; yet, there should always be a modest room for exploratory behavior!",
wraplength=1000,
font_size=12,
row=4,
column=1,
height=3)
# add frame and animation
self.animation_frame = tk.Frame(self)
self.animation_frame.rowconfigure(0, weight=7)
self.animation_frame.rowconfigure(1, weight=1)
self.animation_frame.columnconfigure(0, weight=1)
self.animation_frame.columnconfigure(1, weight=1)
self.animation_frame.columnconfigure(2, weight=1)
self.video_animation = va.video_animation(
self.animation_frame, 0, 0, 0, 0, 1, 0, 240, 180,
"media/module_q/q_learning_algorithm_exploration_v_exploitation_frame/epsilon.mp4",
True, self)
self.epsilon_value_board = tk.Label(self.animation_frame,
textvariable=self.epsilon_value,
font=("Courier", 16))
self.epsilon_value_board.grid(row=0, column=1, rowspan=2)
self.animation_text_frame = tk.Frame(self.animation_frame)
self.animation_text_frame.grid(row=0, column=2, rowspan=2)
self.animation_text_frame.rowconfigure(0, weight=1)
self.animation_text_frame.rowconfigure(1, weight=1)
self.text5 = eh.toggle_label(
self.animation_text_frame,
"Let's see the annealing process in action. Here, we introduce a variable called epsilon. With epsilon probability, we will randomly choose our actions rather than following our action-value system. For this example, we will start the epsilon at 100%, and eventually 'cool it down' to the 5% plateau.",
wraplength=650,
font_size=12,
row=0,
column=0,
width=100,
height=5)
self.text6 = eh.toggle_label(
self.animation_text_frame,
"By way of exploration, the Q-Learning Agent here discovers a new way of hitting the ball, and this new method is effective in eventually getting ball past its opponent, yielding a reward of 100.",
wraplength=650,
font_size=12,
row=1,
column=0,
width=100,
height=5)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.text1.grid(row=1, column=1)
self.animation_frame.grid(row=5, column=1)
self.previous_frame_button.grid(row=6, column=0, padx=0, sticky=tk.W)
self.to_roadmap_button.grid(row=6, column=1, columnspan=1)
self.next_frame_button.grid(row=6, column=2, padx=0, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=4)
self.columnconfigure(2, weight=4)
self.columnconfigure(3, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
self.rowconfigure(7, weight=1)
explanation_width = 50
explanation_height = 2
explanation_wraplength = 400
# add widgets
self.title = tk.Label(self,
text="Action-Value Function: Reward",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
wraplength=1200,
justify=tk.LEFT,
text=
"Okay, let's assume we are on the current (time = t) 'state'. We have three choices of action: go up, down, or stay still. According to our strategy [pop up], what action should we take?"
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
play.strategy_from_pong_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
action_value_function_max_q_frame),
text="Next",
font=("Courier", 14))
# define scheme frame
self.scheme_frame = tk.Frame(self)
self.scheme_frame.columnconfigure(0, weight=1)
self.scheme_frame.columnconfigure(1, weight=1)
self.scheme_frame.columnconfigure(2, weight=1)
self.scheme_frame.rowconfigure(0, weight=1)
self.scheme_frame.rowconfigure(1, weight=1)
self.scheme_frame.rowconfigure(2, weight=1)
self.scheme_frame.rowconfigure(3, weight=1)
self.time_now = tk.Label(self.scheme_frame,
text="time = t",
font=("Courier", 14))
self.time_plus_one = tk.Label(self.scheme_frame,
text="time = t + 1",
font=("Courier", 14))
state_width = 210
state_height = 150
state_image_scale = 0.1
original_image_path = 'media/module_q/action_value_function_reward_frame/original.png'
self.original_image = ImageTk.PhotoImage(
im.scale_image(Image.open(original_image_path), state_image_scale))
self.original_state = tk.Label(self.scheme_frame,
width=state_width,
height=state_height,
image=self.original_image)
self.time_now.grid(row=0, column=0)
self.time_plus_one.grid(row=0, column=2)
self.original_state.grid(row=2, column=0, sticky=tk.E)
# add best_action_scheme (args: self, master, frame_id, scheme_frame, state_image_scale, button_one_row, button_one_column, button_two_row, button_two_column, button_three_row, button_three_column, state_one_row, state_one_column, state_two_row, state_two_column, state_three_row, state_three_column, state_width, state_height, explanation_row, explanation_column, explanation_width, explanation_height, explanation_wraplength, explanation_string)
self.best_action_scheme = eh.best_action_scheme(
self, "reward", self.scheme_frame, state_image_scale, 1, 1, 2, 1,
3, 1, 1, 2, 2, 2, 3, 2, state_width, state_height, 2, 2,
explanation_width, explanation_height, explanation_wraplength,
"According to our strategy, it made sense to press up, because it resulted in a state that awarded a reward of 10."
)
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
function_image_scale = 0.5
function_image_path = 'media/module_q/action_value_function_reward_frame/function.png'
self.function_image = ImageTk.PhotoImage(
im.scale_image(Image.open(function_image_path),
function_image_scale))
self.explanation_two = eh.toggle_label(
self,
"Then, for our Q-Learning algorithm, how about we assign a value to an action, or an action-value (otherwise known as a Q-value) at each possible state?",
wraplength=explanation_wraplength,
row=3,
column=2,
width=explanation_width,
height=explanation_height)
self.explanation_three = eh.toggle_label(
self,
"How so? Well, the simplest way to define such an action-value might be to let it be equal to the reward at the resulting state?:",
wraplength=explanation_wraplength,
row=4,
column=2,
width=explanation_width,
height=explanation_height)
self.explanation_four = eh.toggle_image(self,
image=self.function_image,
row=5,
column=2,
width=400,
height=50)
self.explanation_five = eh.toggle_label(
self,
"And now, we simply choose to take the action with the largest action-value! We now have an algorithm: a way of deciding which action to take in a given state.",
wraplength=explanation_wraplength,
row=6,
column=2,
width=explanation_width,
height=explanation_height)
# lay widgets
self.title.grid(row=0, column=0, columnspan=4)
self.text1.grid(row=1, column=0, columnspan=4)
self.scheme_frame.grid(row=2, column=1, rowspan=5, padx=40)
self.previous_frame_button.grid(row=7, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=7, column=1, columnspan=2)
self.next_frame_button.grid(row=7, column=3, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=4)
self.columnconfigure(2, weight=4)
self.columnconfigure(3, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
self.rowconfigure(6, weight=1)
self.rowconfigure(7, weight=1)
self.rowconfigure(8, weight=1)
explanation_width = 50
explanation_height = 2
explanation_wraplength = 400
# add widgets
self.title = tk.Label(self,
text="Action-Value Function: Max Q",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
wraplength=1200,
justify=tk.LEFT,
text=
"How about now, a little further away? Again we have three choices: according to our strategy [pop up], which action should we take?"
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
action_value_function_reward_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
q_learning_algorithm_initialization_frame),
text="Next",
font=("Courier", 14))
# define scheme frame
self.scheme_frame = tk.Frame(self)
self.scheme_frame.columnconfigure(0, weight=1)
self.scheme_frame.columnconfigure(1, weight=1)
self.scheme_frame.columnconfigure(2, weight=1)
self.scheme_frame.rowconfigure(0, weight=1)
self.scheme_frame.rowconfigure(1, weight=1)
self.scheme_frame.rowconfigure(2, weight=1)
self.scheme_frame.rowconfigure(3, weight=1)
self.time_now = tk.Label(self.scheme_frame,
text="time = t",
font=("Courier", 14))
self.time_plus_one = tk.Label(self.scheme_frame,
text="time = t + 1",
font=("Courier", 14))
state_width = 210
state_height = 150
state_image_scale = 0.1
original_image_path = 'media/module_q/action_value_function_max_q_frame/original.png'
self.original_image = ImageTk.PhotoImage(
im.scale_image(Image.open(original_image_path), state_image_scale))
self.original_state = tk.Label(self.scheme_frame,
width=state_width,
height=state_height,
image=self.original_image)
self.time_now.grid(row=0, column=0)
self.time_plus_one.grid(row=0, column=2)
self.original_state.grid(row=2, column=0, sticky=tk.E)
# add best_action_scheme (args: self, master, frame_id, scheme_frame, state_image_scale, button_one_row, button_one_column, button_two_row, button_two_column, button_three_row, button_three_column, state_one_row, state_one_column, state_two_row, state_two_column, state_three_row, state_three_column, state_width, state_height, explanation_row, explanation_column, explanation_width, explanation_height, explanation_wraplength, explanation_string)
self.best_action_scheme = eh.best_action_scheme(
self, "max_q", self.scheme_frame, state_image_scale, 1, 1, 2, 1, 3,
1, 1, 2, 2, 2, 3, 2, state_width, state_height, 2, 2,
explanation_width, explanation_height, explanation_wraplength,
"According to our strategy, it made sense to press up, because the resulting state gets us closer to the state that would give us a positive reward. Note that this requires some forward thinking."
)
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
function_image_scale = 0.5
function_image_path = 'media/module_q/action_value_function_max_q_frame/function.png'
self.function_image = ImageTk.PhotoImage(
im.scale_image(Image.open(function_image_path),
function_image_scale))
self.explanation_two = eh.toggle_label(
self,
"Unfortunately, our current action-value (Q-value) function does not work here because every reward of our immediate resulting states are all the same at 0.",
wraplength=explanation_wraplength,
row=3,
column=2,
width=explanation_width,
height=explanation_height)
self.explanation_three = eh.toggle_label(
self,
"Indeed, our current Q-value function does not have any notion of forward-planning. Then, how do we give our function this necessary sense of directionality?",
wraplength=explanation_wraplength,
row=4,
column=2,
width=explanation_width,
height=explanation_height)
self.explanation_four = eh.toggle_label(
self,
"The answer is simple: have our action-value function take into account any potential future reward! Then on top of what we already have, let's add the maximum available action-value at the resulting state to our function:",
wraplength=explanation_wraplength,
row=5,
column=2,
width=explanation_width,
height=3)
self.explanation_five = eh.toggle_image(self,
image=self.function_image,
row=6,
column=2,
width=400,
height=50)
self.explanation_six = eh.toggle_label(
self,
"We now have our 'Q-Learning' algorithm!",
wraplength=explanation_wraplength,
row=7,
column=2,
width=explanation_width,
height=explanation_height)
# lay widgets
self.title.grid(row=0, column=0, columnspan=4)
self.text1.grid(row=1, column=0, columnspan=4)
self.scheme_frame.grid(row=2, column=1, rowspan=5, padx=40)
self.previous_frame_button.grid(row=8, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=8, column=1, columnspan=2)
self.next_frame_button.grid(row=8, column=3, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
# add widgets
self.title = tk.Label(self,
text="The Large Problem With Pixel Input",
font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
wraplength=1000,
justify=tk.LEFT,
text=
"But right from the beginning, we encounter a very large problem. Literally. Think about the input that our computer player receives. The information from the game that our 'Q-Learning' player receives is a 600x400 pixel gameplay screen. Assuming that each pixel can be either white or black, that leaves our computer player with a total of 2600 x 400 state spaces to work with."
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
q.action_value_function_revisited_discount_factor_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(q_network_frame),
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self,
"So, when our Q-Learning Agent iterates through every possible (state, action) pair and initialize each action-value as 0, it would have to do it 2600 x 400 times!",
wraplength=1000,
font_size=16,
row=2,
column=1)
input_problem_image_scale = 0.64
input_problem_image_path = 'media/module_dqn/the_large_problem_with_pixel_input_frame/input_problem.png'
self.input_problem_image = ImageTk.PhotoImage(
im.scale_image(Image.open(input_problem_image_path),
input_problem_image_scale))
self.image_one = eh.toggle_image(self,
image=self.input_problem_image,
row=3,
column=1,
width=600,
height=200)
self.text3 = eh.toggle_label(
self,
"Storing pixel data state by state becomes problematic very quickly. For example, if we assume black and white binary pixels, even a 20 x 20 screen would leave us with 2^400 = 2.58 x 10^120 states. To give you some perspective, there are estimated to be around 10^80 atoms in the entire universe.",
wraplength=1000,
font_size=16,
row=4,
column=1)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.text1.grid(row=1, column=1)
self.previous_frame_button.grid(row=5, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=5, column=1, columnspan=1)
self.next_frame_button.grid(row=5, column=7, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
# add widgets
self.title = tk.Label(self, text="Replay Memory", font=("Courier", 48))
self.text1 = tk.Label(
self,
font=("Courier", 16),
justify=tk.LEFT,
wraplength=1000,
text=
"However, our Deep Q-Network algorithm is not without its faults. As it turns out, because it trains the model at every time step, the model is prone to very short sighted reflection, meaning that it doesn't look far back in the past to learn the best way of choosing its actions."
)
self.previous_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(q_network_training_sequence_frame),
self.controller.frames[q_network_training_sequence_frame
].video_animation.open_video()
],
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
challenge.challenge_dqn_player_frame),
text="Next",
font=("Courier", 14))
# add + lay toggle widgets (args: self, master, text_string, width=1, height=1, wraplength=None, justify=tk.LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self,
"Let us say for example that after a good amount of updating and training, our Deep Q-Learning algorithm comes across by chance two non-trivial action-value update instances back to back while pressing down in the bottom corner. This may cause the algorithm to develop a bias towards pursuing this seemingly rewarding (state, action) pair, which is certainly not a winning strategy.",
row=2,
column=1,
wraplength=1000)
self.text3 = eh.toggle_label(
self,
"In 2013, Google's Deepmind developed a technique to effectively combat this short term bias, and it was achieved by introducing the concept of replay memory. Quite simply, replay memory is a storage of action-observation information that goes far back in the future, and every training iteration trains over a 'batch' of randomly chosen episodes from that replay memory, rather than each immediate past episodes.",
row=3,
column=1,
wraplength=1000)
self.text4 = eh.toggle_label(
self,
"Congratulations, we now have our fully-functional Deep Q-Network algorithm!",
row=4,
column=1,
wraplength=1000)
# lay widgets
self.title.grid(row=0, column=0, columnspan=8)
self.text1.grid(row=1, column=1)
self.previous_frame_button.grid(row=5, column=0, padx=10, sticky=tk.W)
self.to_roadmap_button.grid(row=5, column=1, columnspan=1)
self.next_frame_button.grid(row=5, column=2, padx=10, sticky=tk.E)
def create_widgets(self):
# set column and row spacing for the frame
self.columnconfigure(0, weight=1)
self.columnconfigure(1, weight=1)
self.columnconfigure(2, weight=1)
self.rowconfigure(0, weight=1)
self.rowconfigure(1, weight=1)
self.rowconfigure(2, weight=1)
self.rowconfigure(3, weight=1)
self.rowconfigure(4, weight=1)
self.rowconfigure(5, weight=1)
# add widgets
self.title = tk.Label(self, text="Q-Network", font=("Courier", 48))
self.row_one_frame = tk.Frame(self)
self.text1 = tk.Label(
self.row_one_frame,
font=("Courier", 16),
wraplength=500,
justify=tk.LEFT,
text=
"Ok, then. Let's change our strategy and go back to how we think. Remember our strategy? Let's take a look at its language."
)
self.previous_frame_button = tk.Button(
self,
command=lambda: self.controller.show_frame(
the_large_problem_with_pixel_input_frame),
text="Previous",
font=("Courier", 14))
self.to_roadmap_button = tk.Button(
self,
command=lambda: self.controller.show_frame(prelude.roadmap_frame),
text="To Roadmap",
font=("Courier", 14))
self.next_frame_button = tk.Button(
self,
command=lambda: [
self.controller.show_frame(q_network_training_sequence_frame),
self.controller.frames[q_network_training_sequence_frame
].video_animation.open_video()
],
text="Next",
font=("Courier", 14))
# add toggle texts (args: self, master, text_string, width=1, height=1, wraplength=None, justify=LEFT, row=0, column=0, rowspan=1, columnspan=1, font_type="Courier", font_size=16)
self.text2 = eh.toggle_label(
self.row_one_frame,
"- Where the ball is going to end up when it gets to my paddle, based on current trajectory\n- Where my paddle is currently at",
wraplength=500,
font_size=16,
row=0,
column=1,
width=50)
self.text3 = eh.toggle_label(
self,
"So, we were never thinking about the pixel input of each frame to begin with. Rather, we were identifying the characteristics of various key objects in the screen and calculating what that meant for our ability to gain reward.",
wraplength=1000,
font_size=16,
row=2,
column=1,
width=80)
self.text4 = eh.toggle_label(
self,
"And this is exactly where neural networks come in for the rescue! Neural network is meant to predict values from an input.",
wraplength=1000,
font_size=16,
row=3,
column=1)
q_network_image_scale = 0.5
q_network_image_path = 'media/module_dqn/q_network_frame/q_network.png'
self.q_network_image = ImageTk.PhotoImage(
im.scale_image(Image.open(q_network_image_path),
q_network_image_scale))
self.q_network = eh.toggle_image(self,
image=self.q_network_image,
row=4,
column=1,
width=800,
height=300)
# lay widgets
self.title.grid(row=0, column=0, columnspan=3)
self.row_one_frame.grid(row=1, column=1)
self.text1.grid(row=0, column=0)
self.previous_frame_button.grid(row=5, column=0, padx=0, sticky=tk.W)
self.to_roadmap_button.grid(row=5, column=1, columnspan=1)
self.next_frame_button.grid(row=5, column=2, padx=0, sticky=tk.E)
