def test_execute(self, env, fields, dom, dom_elem):
# Test 1 same col
# (left, width) pairs
included = [(101, 5), (99, 5), (100, 0), (105, 0), (99, 1), (80, 40),
(101, 1)]
for (left, width) in included:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["left"] = left
new_dom["children"][1]["width"] = width
new_state = MiniWoBState("utt", None, new_dom)
same_col = SameColToken(
MockReturnElementSet(new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_col.execute(env)
assert isinstance(result, same_col.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert set([result[0].ref]) == set([2])
# Test that these points aren't included
excluded = [(99, 0), (98, 1), (79, 20), (106, 0), (106, 100)]
for (left, width) in excluded:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["left"] = left
new_dom["children"][1]["width"] = width
new_state = MiniWoBState("utt", None, new_dom)
same_col = SameColToken(
MockReturnElementSet(new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_col.execute(env)
assert isinstance(result, same_col.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 0
# Test that you can have multiple neighbors
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["left"] = 103
new_dom["children"][1]["width"] = 10
new_dom["children"][2]["left"] = 99
new_dom["children"][2]["width"] = 10
new_state = MiniWoBState("utt", None, new_dom)
same_col = SameColToken(MockReturnElementSet(
new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_col.execute(env)
assert isinstance(result, same_col.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 2
def test_valid_strings(self, env, dom, dom_elem):
valid_strings = set([
"1 2 3", "2 3 4", "3 4 5", "4 5 6", "1 2", "2 3", "3 4", "4 5",
"5 6", "1", "2", "3", "4", "5", "6", "ONE", "TWO", "text"
])
assert valid_strings == env.valid_strings
# Make a new observation
env.set_last(dom_elem)
new_dom = copy.deepcopy(dom)
new_dom["children"][0]["children"][0]["children"][3]["text"] = \
"A B C D"
new_state = MiniWoBState("new utterance", None, new_dom)
env.observe(new_state)
valid_strings.add("A B C")
valid_strings.add("B C D")
valid_strings.add("A B")
valid_strings.add("B C")
valid_strings.add("C D")
valid_strings.add("A")
valid_strings.add("B")
valid_strings.add("C")
valid_strings.add("D")
# Check updated valid_strings
assert valid_strings == env.valid_strings
def test_utterance(self, env, dom, dom_elem):
# Test that the utterance is correct.
assert env.tokens == word_tokenize("This is the Utterance.")
# Update utterance.
env.set_last(dom_elem)
new_state = MiniWoBState("new utterance", None, dom)
env.observe(new_state)
# Test new value
assert env.tokens == word_tokenize("new utterance")
def get_state(self):
"""Get the current state.
Returns:
MiniWoBState
"""
# Get the utterance
response = self.driver.execute_script('return core.getUtterance();')
if isinstance(response, dict):
utterance = response['utterance']
fields = Fields(response['fields'])
else:
utterance = response
fields = self.field_extractor(utterance)
# Get the DOM
dom_info = self.driver.execute_script('return core.getDOMInfo();')
state = MiniWoBState(utterance, fields, dom_info)
# Get screenshot if requested
if self.record_screenshots:
img = get_screenshot(self.driver, self.task_width,
self.task_height)
state.set_screenshot(img)
return state
def test_text(self, env, dom, dom_elem):
# Test that text elements are reported
text = env.text
assert len(text) == 1
assert text[0].text == "text"
# Make a new observation
env.set_last(dom_elem)
new_dom = copy.deepcopy(dom)
new_dom["children"][0]["children"][0]["children"][2]["text"] = "Hello"
new_state = MiniWoBState("new utterance", None, new_dom)
env.observe(new_state)
# Make sure changes are reflected
text = env.text
assert len(text) == 1
assert text[0].text == "Hello"
def test_buttons(self, env, dom, dom_elem):
# Test that buttons are reported
buttons = env.buttons
assert len(buttons) == 2
text = sorted([buttons[0].text, buttons[1].text])
assert text == ["ONE.", "TWO;"]
# Make a new observation
env.set_last(dom_elem)
new_dom = copy.deepcopy(dom)
new_dom["children"][0]["children"][0]["children"][0]["text"] = "Hello"
new_state = MiniWoBState("new utterance", None, new_dom)
env.observe(new_state)
# Make sure changes are reflected
buttons = env.buttons
assert len(buttons) == 2
text = sorted([buttons[0].text, buttons[1].text])
assert text == ["Hello", "TWO;"]
def test_last(self, env, dom, dom_elem):
# Last should not be set yet.
with pytest.raises(Exception):
env.last
# Set last and check
env.set_last(dom_elem)
assert env.last == dom_elem
# Check can't double update last
with pytest.raises(Exception):
env.set_last(dom_elem)
# Set a new observation
state = MiniWoBState("utt", None, dom)
# Check can't double update observation
env.observe(state)
with pytest.raises(Exception):
env.observe(state)
def test_input(self, env, dom, dom_elem):
# Test that env recognizes all types of input elems
input_elems = env.input
assert len(input_elems) == 3
assert input_elems[0].text == ""
assert input_elems[1].text == ""
assert input_elems[2].text == ""
# Make a new observation
env.set_last(dom_elem)
new_dom = copy.deepcopy(dom)
new_dom["children"][0]["children"][0]["children"][3]["text"] = "Hello"
new_state = MiniWoBState("new utterance", None, new_dom)
env.observe(new_state)
# Make sure changes are reflected
input_elems = env.input
assert len(input_elems) == 3
text = "".join(
[input_elems[0].text, input_elems[1].text, input_elems[2].text])
assert text == "Hello"
def test_execute(self, env, fields, dom, dom_elem):
# Test from StringToken
like = LikeToken(StringToken(u"ONE"))
result = like.execute(env)
assert isinstance(result, like.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert result[0].text == "ONE."
# Test from UtteranceSelectorToken
env.set_last(dom_elem)
new_state = MiniWoBState("TWO utterance blah", None, dom)
env.observe(new_state)
like = LikeToken(UtteranceSelectorToken(0, 1))
result = like.execute(env)
assert isinstance(result, like.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert result[0].text == "TWO;"
# Test return none
like = LikeToken(StringToken(u"abcd"))
result = like.execute(env)
assert isinstance(result, like.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 0
# Test punctuation and whitespace
like = LikeToken(StringToken(u"T ?W O ."))
result = like.execute(env)
assert isinstance(result, like.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert result[0].text == "TWO;"
def test_execute(self, env, fields, dom, dom_elem):
# Three button simple DOM
dom = {
unicode("top"):
0,
unicode("height"):
0,
unicode("width"):
0,
unicode("tag"):
unicode("BODY"),
unicode("ref"):
0,
unicode("left"):
0,
unicode("children"): [{
unicode("top"): 100,
unicode("left"): 100,
unicode("height"): 5,
unicode("width"): 5,
unicode("ref"): 1,
unicode("tag"): unicode("BUTTON"),
unicode("text"): unicode("ONE"),
unicode("children"): []
}, {
unicode("top"): 0,
unicode("left"): 210,
unicode("height"): 5,
unicode("width"): 5,
unicode("ref"): 2,
unicode("tag"): unicode("BUTTON"),
unicode("text"): unicode("TWO"),
unicode("children"): []
}, {
unicode("top"): 0,
unicode("left"): 0,
unicode("height"): 5,
unicode("width"): 5,
unicode("ref"): 3,
unicode("tag"): unicode("BUTTON"),
unicode("text"): unicode("THREE"),
unicode("children"): []
}]
}
env = ExecutionEnvironment(MiniWoBState("utt", None, dom))
dom_elem = env.buttons[0]
# Check all the pts that are near ONE
included_points = [
(75, 95),
(75, 105), # Edge on left
(95, 125),
(105, 125), # Edge on bottom
(125, 105),
(125, 95), # Edge on right
(95, 75),
(105, 75), # Edge on top
(100, 100), # Full overlap
(95, 95),
(105, 95), # Overlap top corners
(95, 105),
(105, 105), # Overlap bottom corners
(81, 81), # Top left corner
(119, 81), # Top right corner
(119, 119), # Bottom right corner
(81, 119), # Bottom left corner
]
# Check that all of the included pts are there
for (left, top) in included_points:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["left"] = left
new_dom["children"][1]["top"] = top
new_state = MiniWoBState("utt", None, new_dom)
near = NearToken(MockReturnElementSet(new_state.dom.children[0]))
env.set_last(dom_elem)
env.observe(new_state)
dom_elem = env.buttons[0]
result = near.execute(env)
assert isinstance(result, near.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert set([result[0].ref]) == set([2])
excluded_points = [
(74, 95),
(75, 106), # Edge on left
(94, 125),
(105, 126), # Edge on bottom
(125, 106),
(126, 95), # Edge on right
(95, 74),
(106, 75), # Edge on top
(0, 0), # Not even close
(80, 81), # Top left corner
(119, 80), # Top right corner
(120, 119), # Bottom right corner
(81, 120), # Bottom left corner
]
# Check that all of the excluded pts are not there
for (left, top) in excluded_points:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["left"] = left
new_dom["children"][1]["top"] = top
new_state = MiniWoBState("utt", None, new_dom)
near = NearToken(MockReturnElementSet(new_state.dom.children[0]))
env.set_last(dom_elem)
env.observe(new_state)
dom_elem = env.buttons[0]
result = near.execute(env)
assert isinstance(result, near.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 0
# Check that you can have multiple pts in there
new_dom = copy.deepcopy(dom)
left, top = included_points[0]
new_dom["children"][1]["left"] = left
new_dom["children"][1]["top"] = top
left, top = included_points[1]
new_dom["children"][2]["left"] = left
new_dom["children"][2]["top"] = top
new_state = MiniWoBState("utt", None, new_dom)
near = NearToken(MockReturnElementSet(new_state.dom.children[0]))
env.set_last(dom_elem)
env.observe(new_state)
result = near.execute(env)
assert isinstance(result, near.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 2
assert set([result[0].ref, result[1].ref]) == set([2, 3])
# Check for more than one input element
near = NearToken(ButtonsToken())
result = near.execute(env)
assert isinstance(result, near.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 3
assert set([result[0].ref, result[1].ref,
result[2].ref]) == set([1, 2, 3])
def test_execute(self, env, fields, dom, dom_elem):
# TODO: Refactor these tests (put the loop into own method)
# Test 1 same row
# (y, height) pairs
included = [(101, 5), (99, 5), (100, 0), (105, 0), (99, 1), (80, 40),
(101, 1)]
for (top, height) in included:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["top"] = top
new_dom["children"][1]["height"] = height
new_state = MiniWoBState("utt", None, new_dom)
same_row = SameRowToken(
MockReturnElementSet(new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_row.execute(env)
assert isinstance(result, same_row.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 1
assert set([result[0].ref]) == set([2])
# Test that these points aren't included
excluded = [(99, 0), (98, 1), (79, 20), (106, 0), (106, 100)]
for (top, height) in excluded:
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["top"] = top
new_dom["children"][1]["height"] = height
new_state = MiniWoBState("utt", None, new_dom)
same_row = SameRowToken(
MockReturnElementSet(new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_row.execute(env)
assert isinstance(result, same_row.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 0
# Test that you can have multiple neighbors
new_dom = copy.deepcopy(dom)
new_dom["children"][1]["top"] = 103
new_dom["children"][1]["height"] = 10
new_dom["children"][2]["top"] = 99
new_dom["children"][2]["height"] = 10
new_state = MiniWoBState("utt", None, new_dom)
same_row = SameRowToken(MockReturnElementSet(
new_state.dom.children[0]))
dom_elem = env.buttons[0]
env.set_last(dom_elem)
env.observe(new_state)
result = same_row.execute(env)
assert isinstance(result, same_row.return_type)
assert isinstance(result, ElementSet)
assert len(result) == 2
def test_execute(self, env, fields, dom, dom_elem):
# Test input
input_token = InputElementsToken()
result = input_token.execute(env)
assert env.input == result
assert isinstance(result, input_token.return_type)
assert isinstance(result, ElementSet)
# Test buttons
buttons_token = ButtonsToken()
result = buttons_token.execute(env)
assert env.buttons == result
assert isinstance(result, buttons_token.return_type)
assert isinstance(result, ElementSet)
# Test text
text_token = TextToken()
result = text_token.execute(env)
assert env.text == result
assert isinstance(result, text_token.return_type)
assert isinstance(result, ElementSet)
# Reset DOM
env.set_last(dom_elem)
new_dom = {
unicode("top"): 0,
unicode("height"): 210,
unicode("width"): 500,
unicode("tag"): unicode("BODY"),
unicode("ref"): 30,
unicode("children"): [],
unicode("left"): 0
}
new_state = MiniWoBState("new utterance", None, new_dom)
env.observe(new_state)
# Test input
input_token = InputElementsToken()
result = input_token.execute(env)
assert env.input == result
assert isinstance(result, input_token.return_type)
assert isinstance(result, ElementSet)
# Test buttons
buttons_token = ButtonsToken()
result = buttons_token.execute(env)
assert env.buttons == result
assert isinstance(result, buttons_token.return_type)
assert isinstance(result, ElementSet)
# Test text
text_token = TextToken()
result = text_token.execute(env)
assert env.text == result
assert isinstance(result, text_token.return_type)
assert isinstance(result, ElementSet)
# Test last
last_token = LastToken()
result = last_token.execute(env)
assert ElementSet([dom_elem]) == result
assert isinstance(result, last_token.return_type)
assert isinstance(result, ElementSet)
def env(self, dom, fields):
state = MiniWoBState(u"This is the Utterance.", fields, dom)
env = ExecutionEnvironment(state)
return env
def dom_elem(self, dom):
state = MiniWoBState("", None, dom)
return state.dom_elements[-1]
def _chunk_events(self, raw_state_pairs, utterance, fields):
"""Find chunks of events that express a single action."""
chunks = []
last_mousedown = None
last_mouseup = None
last_keydown = None
# Current modifier keys (shift, ctrl, alt)
current_modifiers = set()
# Number of keypresses left to be checked with keydowns
pending_keypresses = 0
for i, (raw_state, raw_state_after) in enumerate(raw_state_pairs):
raw_action = raw_state['action']
state = MiniWoBState(utterance, fields, raw_state['dom'])
target = self._target(state.dom_elements)
t = raw_action['type']
if t == 'mousedown':
if last_mousedown:
logging.warning('Two consecutive mousedowns @ %d', i)
# Click is missing; convert the last mousedown into a click
chunks.append(
Chunk('click', last_mousedown.state,
last_mousedown.target, last_mousedown.args))
last_mousedown = last_mouseup = None
coord = (raw_action['x'], raw_action['y'])
last_mousedown = Chunk('mousedown', state, target, coord)
elif t == 'mouseup':
assert last_mousedown, 'Cannot have mouseup without mousedown @ {}'.format(
i)
assert not last_mouseup, 'Two consecutive mouseups @ {}'.format(
i)
coord = (raw_action['x'], raw_action['y'])
last_mouseup = Chunk('mouseup', state, target, coord)
elif t == 'click':
if last_mouseup:
# TODO: Currently dragging is interpreted as clicking
chunks.append(
Chunk('click', last_mousedown.state,
last_mousedown.target, last_mousedown.args))
last_mousedown = last_mouseup = None
else:
# Spurious click event from <label> tag
pass
elif t == 'dblclick':
# dblclick is ignored (two clicks are already in the chunk list)
continue
elif t == 'keydown':
keycode = raw_action['keyCode']
if keycode in EpisodeGraph.MODIFIERS:
current_modifiers.add(keycode)
else:
last_keydown = Chunk(
'keydown', state, target,
sorted(list(current_modifiers) + [keycode]))
elif t == 'keyup':
keycode = raw_action['keyCode']
if keycode in EpisodeGraph.MODIFIERS:
assert keycode in current_modifiers,\
'keyup on modifier without keydown @ {}'.format(i)
current_modifiers.remove(keycode)
elif pending_keypresses:
pending_keypresses -= 1
else:
assert last_keydown,\
'keyup without keydown @ {}'.format(i)
# Hotkey
state_after = MiniWoBState(utterance, fields,
raw_state_after['dom'])
chunk = self._resolve_hotkey(last_keydown.state, target,
last_keydown.args,
state_after)
if chunk:
chunks.append(chunk)
elif t == 'keypress':
char = chr(raw_action['charCode'])
chunks.append(
Chunk('type', last_keydown.state, last_keydown.target,
char))
pending_keypresses += 1
else:
raise ValueError('Unknown action type: {}'.format(t))
return chunks
def _parse_raw_demo_original(self, raw_demo, field_extractor):
"""Takes the raw demo and spits out the relevant states.
Algorithm: Look at mousedown / keypress events
Args:
raw_demo (dict): json contents of demo file
field_extractor (FieldsExtractor): the fields extractor for this task
Returns:
state_vertices (list[StateVertex])
"""
# Filter out only for keypresses and mousedowns (BEFORE)
utterance = raw_demo['utterance']
if 'fields' in raw_demo:
fields = Fields(raw_demo['fields'])
else:
fields = field_extractor(utterance)
raw_states = raw_demo["states"]
state_vertices = []
actions = []
vertex_number = 0
for i, raw_state in enumerate(raw_states[1:]):
raw_action = raw_state["action"]
if raw_action["timing"] == EpisodeGraph.BEFORE:
if raw_action["type"] == "mousedown":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
click = MiniWoBElementClick(target)
state_vertex = StateVertex(
miniwob_state,
[ActionEdge(click, vertex_number, vertex_number + 1)])
state_vertices.append(state_vertex)
vertex_number += 1
elif raw_action["type"] == "keypress":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
char = chr(raw_action["keyCode"])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
type_action = MiniWoBFocusAndType(target, char)
state_vertex = StateVertex(miniwob_state, [
ActionEdge(type_action, vertex_number,
vertex_number + 1)
])
state_vertices.append(state_vertex)
vertex_number += 1
# Collapse consecutive FocusAndTypes into one
for i, vertex in enumerate(state_vertices):
curr_action = vertex.action_edges[0].action
if not isinstance(curr_action, MiniWoBFocusAndType):
continue
aggregated_text = curr_action.text
while i + 1 < len(state_vertices):
next_action = state_vertices[i + 1].action_edges[0].action
if not isinstance(next_action, MiniWoBFocusAndType) or \
curr_action.element != next_action.element:
break
aggregated_text += next_action.text
del next_action
del state_vertices[i + 1]
vertex.action_edges[0] = ActionEdge(
MiniWoBFocusAndType(curr_action.element, aggregated_text), i,
i + 1)
# Collapse Click then FocusAndType into just FocusAndType
collapsed_state_vertices = []
for index in range(len(state_vertices) - 1):
curr_action = state_vertices[index].action_edges[0].action
next_action = state_vertices[index + 1].action_edges[0].action
if not(isinstance(curr_action, MiniWoBElementClick) and \
isinstance(next_action, MiniWoBFocusAndType) and \
curr_action.element == next_action.element):
collapsed_state_vertices.append(state_vertices[index])
collapsed_state_vertices.append(state_vertices[-1])
# Correct the edge indices
for i, state_vertex in enumerate(collapsed_state_vertices):
state_vertex.action_edges[0] = ActionEdge(
state_vertex.action_edges[0].action, i, i + 1)
return collapsed_state_vertices