def _find_equivalent_action(self, dom_diff, state_before, state_after):
"""Return a single action that could produce the dom diff.
Args:
dom_diff (list[DOMElement])
state_before (MiniWoBState)
state_after (MiniWoBState)
Returns:
MiniWoBAction or None
"""
if len(dom_diff) > 1:
return
ref = dom_diff[0].ref
elt_before = self._element_with_ref(state_before, ref)
elt_after = self._element_with_ref(state_after, ref)
if not elt_before or not elt_after:
return
# Click
if (elt_before.value == elt_after.value and not elt_before.tampered
and elt_after.tampered):
return MiniWoBElementClick(elt_before)
if elt_before.value != elt_after.value:
if elt_before.tag in ('input_checkbox', 'input_radio'):
return MiniWoBElementClick(elt_before)
else:
return MiniWoBFocusAndType(elt_before, elt_after.value)
def execute(self, env):
elements = self._parameter.execute(env)
element = elements.sample_non_text()
# Update last
env.set_last(element)
return MiniWoBElementClick(element)
def get_element_click(self, state):
"""Get the action that clicks the button."""
for element in state.dom_elements:
if element.tag == 'button':
action = MiniWoBElementClick(element, fail_hard=True)
print 'Clicking with {}'.format(action)
return action
raise ValueError('Cannot find button: {}'.format(str(state.dom_elements)))
def get_input_click(self, state):
"""Get the action that clicks an input element."""
for element in state.dom_elements:
if element.tag == 'input_text':
action = MiniWoBElementClick(element, fail_hard=True)
print('Clicking with {}'.format(action))
return action
raise ValueError('Cannot find input: {}'.format(str(
state.dom_elements)))
def _find_shortcuts(self, state_vertices):
"""Takes the list of StateVertex and finds ActionEdges between
non-consecutive state vertices and adds these edges to the passed StateVertexs
Modifies edges in state_vertices directly
Args:
state_vertices (list[StateVertex])
"""
# Single step:
for i, vi in enumerate(state_vertices):
is_last = (i == len(state_vertices) - 1)
for action_edge in vi.action_edges[:]:
dom_diff = True if is_last else self._dom_diff(
vi.state.dom, state_vertices[i + 1].state.dom)
if self._logfile and (dom_diff is True or len(dom_diff) <= 5):
print >> self._logfile, 'DIFF', i, ':', dom_diff
if not dom_diff:
vi.action_edges.append(ActionEdge(None, i, i + 1))
elif isinstance(action_edge, DummyActionEdge):
chunk = action_edge.chunk
if action_edge.reason == 'nonleaf':
# Action on non-leaf: Try all leaves instead
leaves = self._get_leaves(chunk.target)
if len(leaves) <= self.MAX_LEAVES:
for leaf in self._get_leaves(chunk.target):
if chunk.action == 'click':
action = MiniWoBElementClick(leaf)
elif chunk.action == 'type':
action = MiniWoBFocusAndType(leaf, chunk.args)
else:
raise ValueError('Invalid nonleaf DummyActionEdge')
vi.action_edges.append(ActionEdge(action, i, i + 1))
elif not is_last:
action = self._find_equivalent_action(
dom_diff, vi.state, state_vertices[i + 1].state)
if action:
vi.action_edges.append(ActionEdge(action, i, i + 1))
# Multiple steps:
for i in xrange(len(state_vertices)):
vi = state_vertices[i]
for j in xrange(i + 2, min(i + 1 + self.MAX_SHORTCUT_LENGTH, len(state_vertices))):
vj = state_vertices[j]
dom_diff = self._dom_diff(vi.state.dom, vj.state.dom)
if self._logfile and len(dom_diff) <= 5:
print >> self._logfile, 'DIFF', i, '->', j, ':', dom_diff
if not dom_diff:
vi.action_edges.append(ActionEdge(None, i, j))
else:
action = self._find_equivalent_action(dom_diff, vi.state, vj.state)
if action:
vi.action_edges.append(ActionEdge(action, i, j))
def _compute_action_scores(elem_indices, dom_elems, dom_probs,
click_or_type_probs, type_values, type_value_probs,
state_values):
"""Compute action scores (log probabilities).
Args:
dom_elems (list[list[DOMElement]]): a batch of DOMElement lists
(padded to the same length)
dom_probs (Variable): of shape (batch_size, max_dom_elems)
click_or_type_probs (Variable): of shape (batch_size, 2). 0 index
= click, 1 index = type
type_values (list[list[unicode]]): a batch of query values (NOT
padded!)
type_value_probs (Variable): of shape (batch_size, max_values)
state_values (Variable): of shape (batch_size,)
Returns:
action_scores_batch (list[ActionScores])
"""
# shape checks
batch_size = len(dom_elems)
dim1, max_dom_elems = dom_probs.size()
dim2, max_type_values = type_value_probs.size()
dim3, click_or_type_classes = click_or_type_probs.size()
assert dim1 == batch_size
assert dim2 == batch_size
assert dim3 == batch_size
assert click_or_type_classes == 2
assert len(type_values) == batch_size
# check that dom_elems has been appropriately padded
for elems in dom_elems:
assert len(elems) == max_dom_elems
# NOTE: type_values are NOT padded
action_scores_batch = []
for batch_idx in range(batch_size):
action_scores = {}
click_prob, type_prob = click_or_type_probs[batch_idx]
# these are scalar Variables
dom_elems_b, dom_probs_b = dom_elems[batch_idx], dom_probs[batch_idx]
type_values_b, type_value_probs_b = type_values[batch_idx], type_value_probs[batch_idx]
assert len(dom_elems_b) == len(dom_probs_b)
assert len(type_values_b) <= len(type_value_probs_b)
# TODO: HACK :'(
chosen_index = elem_indices[batch_idx]
elem = dom_elems_b[chosen_index]
elem_prob = dom_probs_b[chosen_index]
# elem_prob is a scalar Variable: it has size == (1,)
assert not isinstance(elem, DOMElementPAD)
assert elem.tag != 't'
# generate click action
click_action = MiniWoBElementClick(elem)
action_scores[click_action] = torch.log(click_prob) + \
torch.log(elem_prob)
# generate focus-and-type actions
for type_value, value_prob in zip(type_values_b, type_value_probs_b):
# note that zip truncates to the shorter of its two arguments
type_action = MiniWoBFocusAndType(elem, type_value)
action_scores[type_action] = torch.log(type_prob) + \
torch.log(elem_prob) + torch.log(value_prob)
state_value = state_values[batch_idx] # scalar Variable
action_scores_batch.append(ActionScores(action_scores, state_value))
return action_scores_batch
def _parse_raw_demo_chunk(self,
raw_demo,
field_extractor,
find_shortcuts=False):
"""Takes the raw demo and spits out the relevant states.
Algorithm: Consider each chunk of events that express a single action.
Possible chunks are:
- click (mousedown mouseup click)
- double-click is ignored for now
- drag (mousedown mouseup click, with different coordinates)
- type (keydown* keypress keyup)
- hotkey (keydown* keyup, where keyup is not a modifier key)
Args:
raw_demo (dict): json contents of demo file
field_extractor (FieldsExtractor): the fields extractor for this task
find_shortcuts (bool): whether to also find possible shortcuts
in the graph. If false, the graph will be a sequence.
Returns:
state_vertices (list[StateVertex])
"""
utterance = raw_demo['utterance']
if 'fields' in raw_demo:
fields = Fields(raw_demo['fields'])
else:
fields = field_extractor(utterance)
raw_states = raw_demo['states']
# Group BEFORE and AFTER
# Some AFTER are missing due to event propagation being stopped,
# in which case we also use BEFORE for AFTER
raw_state_pairs = []
current_before = None
for i, raw_state in enumerate(raw_states[1:]):
if raw_state['action']['type'] == 'scroll':
# Skip all scroll actions
continue
if raw_state['action']['timing'] == EpisodeGraph.BEFORE:
if current_before:
# Two consecutive BEFOREs
logging.warning('state %d is BEFORE without AFTER', i - 1)
raw_state_pairs.append((current_before, current_before))
current_before = raw_state
elif raw_state['action']['timing'] == EpisodeGraph.AFTER:
if not current_before:
# Two consecutive AFTERs
logging.warning('state %d is AFTER without BEFORE', i)
current_before = raw_state
raw_state_pairs.append((current_before, raw_state))
current_before = None
if current_before:
# Lingering BEFORE at the end
logging.warning('state %d is BEFORE without AFTER', i - 1)
raw_state_pairs.append((current_before, current_before))
if self._logfile:
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
# print >> self._logfile, '#' * 10, 'PAIRS'
self._logfile.write('Utterance: {}'.format(utterance))
self._logfile.write('Fields: {}'.format(fields))
self._logfile.write('#' * 10 + 'PAIRS')
for i, (s1, s2) in enumerate(raw_state_pairs):
self._logfile.write('@' + str(i) + ':' + str(s1['action']) +
str(s2['action']))
chunks = self._chunk_events(raw_state_pairs, utterance, fields)
chunks = self._collapse_type_actions(chunks)
if self._logfile:
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
# print >> self._logfile, '#' * 10, 'CHUNKS'
self._logfile.write('Utterance: {}'.format(utterance))
self._logfile.write('Fields:{}'.format(fields))
self._logfile.write('#' * 10 + 'CHUNKS')
for i, chunk in enumerate(chunks):
# print >> self._logfile, '@', i, ':', chunk
self._logfile.write('@' + str(i) + ':' + str(chunk))
# Create base vertices
state_vertices = []
for chunk in chunks:
start, end = len(state_vertices), len(state_vertices) + 1
if not chunk.target:
# Probably clicking/dragging on the instruction box
continue
if chunk.action == 'click':
action = MiniWoBElementClick(chunk.target)
if chunk.target.is_leaf:
action_edge = ActionEdge(action, start, end)
else:
action_edge = DummyActionEdge(chunk, start, end, 'nonleaf')
elif chunk.action == 'type':
action = MiniWoBFocusAndType(chunk.target, chunk.args)
if chunk.target.is_leaf:
action_edge = ActionEdge(action, start, end)
else:
action_edge = DummyActionEdge(chunk, start, end, 'nonleaf')
else:
action_edge = DummyActionEdge(chunk, start, end, 'unknown')
# If we don't plan to find shortcuts, we cannot have dummy edges
if not find_shortcuts and isinstance(action_edge, DummyActionEdge):
continue
state_vertex = StateVertex(chunk.state, [action_edge])
state_vertices.append(state_vertex)
if self._logfile:
# print >> self._logfile, '#' * 10, 'GRAPH'
self._logfile.write('#' * 10 + 'GRAPH')
for i, v in enumerate(state_vertices):
self._logfile.write('@' + str(i) + ':' + str(v.action_edges))
self._logfile.write(v.state.dom.visualize())
if find_shortcuts:
if self._logfile:
# print >> self._logfile, '#' * 10, 'SHORTCUTS'
self._logfile.write('#' * 10 + 'SHORTCUTS')
self._find_shortcuts(state_vertices)
# Remove dummy edges
for i, state_vertex in enumerate(state_vertices):
state_vertex.action_edges[:] = [
e for e in state_vertex.action_edges
if not isinstance(e, DummyActionEdge)
]
# To prevent empty states, add skip edges to the next state
if not state_vertex.action_edges:
state_vertex.action_edges.append(ActionEdge(None, i, i + 1))
if self._logfile:
# print >> self._logfile, '#' * 10, 'FINAL'
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
self._logfile.write('#' * 10 + 'FINAL')
self._logfile.write('Utterance:{}'.format(utterance))
self._logfile.write('Fields:{}'.format(fields))
for i, v in enumerate(state_vertices):
# print >> self._logfile, '@', i, ':', v.action_edges
self._logfile.write('@' + str(i) + ':' + str(v.action_edges))
return state_vertices
def _parse_raw_demo_original(self, raw_demo, field_extractor):
"""Takes the raw demo and spits out the relevant states.
Algorithm: Look at mousedown / keypress events
Args:
raw_demo (dict): json contents of demo file
field_extractor (FieldsExtractor): the fields extractor for this task
Returns:
state_vertices (list[StateVertex])
"""
# Filter out only for keypresses and mousedowns (BEFORE)
utterance = raw_demo['utterance']
if 'fields' in raw_demo:
fields = Fields(raw_demo['fields'])
else:
fields = field_extractor(utterance)
raw_states = raw_demo["states"]
state_vertices = []
actions = []
vertex_number = 0
for i, raw_state in enumerate(raw_states[1:]):
raw_action = raw_state["action"]
if raw_action["timing"] == EpisodeGraph.BEFORE:
if raw_action["type"] == "mousedown":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
click = MiniWoBElementClick(target)
state_vertex = StateVertex(
miniwob_state,
[ActionEdge(click, vertex_number, vertex_number + 1)])
state_vertices.append(state_vertex)
vertex_number += 1
elif raw_action["type"] == "keypress":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
char = chr(raw_action["keyCode"])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
type_action = MiniWoBFocusAndType(target, char)
state_vertex = StateVertex(miniwob_state, [
ActionEdge(type_action, vertex_number,
vertex_number + 1)
])
state_vertices.append(state_vertex)
vertex_number += 1
# Collapse consecutive FocusAndTypes into one
for i, vertex in enumerate(state_vertices):
curr_action = vertex.action_edges[0].action
if not isinstance(curr_action, MiniWoBFocusAndType):
continue
aggregated_text = curr_action.text
while i + 1 < len(state_vertices):
next_action = state_vertices[i + 1].action_edges[0].action
if not isinstance(next_action, MiniWoBFocusAndType) or \
curr_action.element != next_action.element:
break
aggregated_text += next_action.text
del next_action
del state_vertices[i + 1]
vertex.action_edges[0] = ActionEdge(
MiniWoBFocusAndType(curr_action.element, aggregated_text), i,
i + 1)
# Collapse Click then FocusAndType into just FocusAndType
collapsed_state_vertices = []
for index in range(len(state_vertices) - 1):
curr_action = state_vertices[index].action_edges[0].action
next_action = state_vertices[index + 1].action_edges[0].action
if not(isinstance(curr_action, MiniWoBElementClick) and \
isinstance(next_action, MiniWoBFocusAndType) and \
curr_action.element == next_action.element):
collapsed_state_vertices.append(state_vertices[index])
collapsed_state_vertices.append(state_vertices[-1])
# Correct the edge indices
for i, state_vertex in enumerate(collapsed_state_vertices):
state_vertex.action_edges[0] = ActionEdge(
state_vertex.action_edges[0].action, i, i + 1)
return collapsed_state_vertices
def create_element_click_action(self, element):
action = MiniWoBElementClick(element, fail_hard=True)
print 'Clicking with {}'.format(action)
return action