def __init__(self, config):
labeled_demos = [
LabeledDemonstration.from_oracle_programs(
[[
WeightedProgram(
ClickToken(LikeToken(FieldsValueSelectorToken(1))), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Reply"))), 1)
],
[
WeightedProgram(
FocusAndTypeToken(InputElementsToken(),
FieldsValueSelectorToken(0)), 1)
],
[WeightedProgram(ClickToken(LikeToken(StringToken(u""))), 1)]
], # TODO: Remove this hack
"Find the email by Harmonia and reply to them with the text \"hello\".",
Fields({
"by": "Harmonia",
"message": "hello"
}))
]
super(EmailInboxReplyOracleProgramPolicy,
self).__init__(labeled_demos, config)
def __init__(self, config):
labeled_demos = [
LabeledDemonstration.from_oracle_programs(
[[
WeightedProgram(
ClickToken(LikeToken(FieldsValueSelectorToken(1))), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Forward"))), 1)
],
[
WeightedProgram(
FocusAndTypeToken(InputElementsToken(),
FieldsValueSelectorToken(0)), 1)
],
[WeightedProgram(ClickToken(LikeToken(StringToken(u""))), 1)]
], # TODO: Remove this hack
"Find the email by Harmonia and forward that email to Eleanore.",
Fields({
"by": "Harmonia",
"to": "Eleanore"
}))
]
super(EmailInboxForwardOracleProgramPolicy,
self).__init__(labeled_demos, config)
def __init__(self, config):
labeled_demos = [
LabeledDemonstration.from_oracle_programs([
[
WeightedProgram(
FocusAndTypeToken(
NearToken(LikeToken(StringToken(u"Username"))),
UtteranceSelectorToken(4, 5)), 1)
],
[
WeightedProgram(
FocusAndTypeToken(
NearToken(LikeToken(StringToken(u"Password"))),
UtteranceSelectorToken(10, 11)), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Login"))), 1)
]
], "Enter the username \"blah\" and the password \"blah\" into the text fields and press login.",
Fields({
"username": "******",
"password": "******"
}))
]
super(LoginUserOracleProgramPolicy,
self).__init__(labeled_demos, config)
def __init__(self, config):
# Appears on first page
labeled_demos = [
LabeledDemonstration.from_oracle_programs([[
WeightedProgram(
ClickToken(LikeToken(UtteranceSelectorToken(12, 13))), 1)
]], "Switch between the tabs to find and click on the link \"x\".",
Fields({"target": "x"}))
]
# Appears in other tabs
labeled_demos += [
LabeledDemonstration.from_oracle_programs(
[[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Tab #2"))), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(UtteranceSelectorToken(12, 13))),
1)
]],
"Switch between the tabs to find and click on the link \"x\".",
Fields({"target": "x"}))
]
labeled_demos += [
LabeledDemonstration.from_oracle_programs(
[[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Tab #2"))), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Tab #3"))), 1)
],
[
WeightedProgram(
ClickToken(LikeToken(UtteranceSelectorToken(12, 13))),
1)
]],
"Switch between the tabs to find and click on the link \"x\".",
Fields({"target": "x"}))
]
super(ClickTab2OracleProgramPolicy,
self).__init__(labeled_demos, config)
def demo(num_boxes, randomize_order):
programs_sequence = [check_field(i + 1) for i in range(num_boxes)]
if randomize_order:
random.shuffle(programs_sequence)
programs_sequence.append(submit())
utt, fields = {
0: ("Click nothing then submit", Fields({"button": "submit"})),
1: ("Click a then submit",
Fields({
"target 0": "a",
"button": "submit"
})),
2: ("Click a,b then submit",
Fields({
"target 0": "a",
"target 1": "b",
"button": "submit"
})),
3: ("Click a,b,c then submit",
Fields({
"target 0": "a",
"target 1": "b",
"target 2": "c",
"button": "submit"
})),
4: ("Click a,b,c,d then submit",
Fields({
"target 0": "a",
"target 1": "b",
"target 2": "c",
"target 3": "d",
"button": "submit"
})),
5: ("Click a,b,c,d,e then submit",
Fields({
"target 0": "a",
"target 1": "b",
"target 2": "c",
"target 3": "d",
"target 4": "e",
"button": "submit"
})),
6: ("Click a,b,c,d,e,f then submit",
Fields({
"target 0": "a",
"target 1": "b",
"target 2": "c",
"target 3": "d",
"target 4": "e",
"target 5": "f",
"button": "submit"
})),
}[num_boxes]
return LabeledDemonstration.from_oracle_programs(
programs_sequence, utt, fields)
def __init__(self, config):
labeled_demos = [
LabeledDemonstration.from_oracle_programs([[
WeightedProgram(
ClickToken(LikeToken(UtteranceSelectorToken(4, 5))), 1)
]], "Switch between the tabs to find and click on the link \"x\".",
Fields({"target": "x"}))
]
super(ClickButtonOracleProgramPolicy,
self).__init__(labeled_demos, config)
def get_state(self):
"""Get the current state.
Returns:
MiniWoBState
"""
# Get the utterance
response = self.driver.execute_script('return core.getUtterance();')
if isinstance(response, dict):
utterance = response['utterance']
fields = Fields(response['fields'])
else:
utterance = response
fields = self.field_extractor(utterance)
# Get the DOM
dom_info = self.driver.execute_script('return core.getDOMInfo();')
state = MiniWoBState(utterance, fields, dom_info)
# Get screenshot if requested
if self.record_screenshots:
img = get_screenshot(self.driver, self.task_width,
self.task_height)
state.set_screenshot(img)
return state
def __init__(self, config):
labeled_demos = [
# block
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# Block
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Block"))), 1)
]
],
"For the user @jess, click on the \"Block\" button.",
Fields({
"user": "******",
"button": "block"
})),
# reply
LabeledDemonstration.from_oracle_programs(
[
# reply
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="reply")), 1)
],
],
"For the user @jess, click on the \"Reply\" button.",
Fields({
"user": "******",
"button": "reply"
})),
# like
LabeledDemonstration.from_oracle_programs(
[
# like
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="like")), 1)
],
],
"For the user @jess, click on the \"Like\" button.",
Fields({
"user": "******",
"button": "like"
})),
# share via DM
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# share
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"share"))), 1)
]
],
"For the user @jess, click on the \"share\" button.",
Fields({
"user": "******",
"button": "share"
})),
# copy
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# copy
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"copy"))), 1)
]
],
"For the user @jess, click on the \"Copy\" button.",
Fields({
"user": "******",
"button": "copy"
})),
# embed
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# embed
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"embed"))), 1)
]
],
"For the user @jess, click on the \"Embed\" button.",
Fields({
"user": "******",
"button": "embed"
})),
# mute
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# mute
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"mute"))), 1)
]
],
"For the user @jess, click on the \"Mute\" button.",
Fields({
"user": "******",
"button": "mute"
})),
# report
LabeledDemonstration.from_oracle_programs(
[
# More
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="more")), 1)
],
# embed
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"report"))), 1)
]
],
"For the user @jess, click on the \"Report\" button.",
Fields({
"user": "******",
"button": "report"
})),
# retweet
LabeledDemonstration.from_oracle_programs(
[
# like
[
WeightedProgram(
ClickToken(
NearToken(LikeToken(
FieldsValueSelectorToken(1)),
classes="retweet")), 1)
],
],
"For the user @jess, click on the \"Retweet\" button.",
Fields({
"user": "******",
"button": "retweet"
})),
]
super(SocialMediaOracle, self).__init__(labeled_demos, config)
def __init__(self, config):
labeled_demos = [
LabeledDemonstration.from_oracle_programs(
[
#######
# forward
# Click name of sender
[
WeightedProgram(
ClickToken(LikeToken(FieldsValueSelectorToken(0))),
1)
],
# Forward
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Forward"))), 1)
],
# Type name of recipient
[
WeightedProgram(
FocusAndTypeToken(InputElementsToken(),
FieldsValueSelectorToken(2)), 1)
],
# Send button
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u""))), 1)
]
],
"Find the email by A and forward that email to B.",
Fields({
"by": "A",
"task": "forward",
"to": "B"
})),
LabeledDemonstration.from_oracle_programs(
[
#######
# reply
# Click name of sender
[
WeightedProgram(
ClickToken(LikeToken(FieldsValueSelectorToken(0))),
1)
],
# Reply
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u"Reply"))), 1)
],
# Type message
[
WeightedProgram(
FocusAndTypeToken(InputElementsToken(),
FieldsValueSelectorToken(1)), 1)
],
# Send button
[
WeightedProgram(
ClickToken(LikeToken(StringToken(u""))), 1)
]
],
"Find the email by A and reply to them with the text \"B\".",
Fields({
"by": "A",
"message": "B",
"task": "reply"
})),
LabeledDemonstration.from_oracle_programs(
[
#######
# trash
# Click name of sender
[
WeightedProgram(
ClickToken(
SameRowToken(
LikeToken(FieldsValueSelectorToken(0)))),
1)
],
],
"Find the email by A and click the trash icon to delete it.",
Fields({
"by": "A",
"task": "delete"
})),
LabeledDemonstration.from_oracle_programs(
[
#######
# star
# Click name of sender
[
WeightedProgram(
ClickToken(
SameRowToken(
LikeToken(FieldsValueSelectorToken(0)))),
1)
],
],
("Find the email by A and click the star "
"icon to mark it as important."),
Fields({
"by": "A",
"task": "star"
})),
]
super(EmailNoScrollOracle, self).__init__(labeled_demos, config)
def fields(self):
return Fields({u"key": u"value", u"key2": u"value2"})
def _alignment_fields(self, states):
"""Returns state utterances wrapped in a Fields (list[Fields])"""
return [Fields({"utterance": state.utterance}) for state in states]
def _parse_raw_demo_chunk(self,
raw_demo,
field_extractor,
find_shortcuts=False):
"""Takes the raw demo and spits out the relevant states.
Algorithm: Consider each chunk of events that express a single action.
Possible chunks are:
- click (mousedown mouseup click)
- double-click is ignored for now
- drag (mousedown mouseup click, with different coordinates)
- type (keydown* keypress keyup)
- hotkey (keydown* keyup, where keyup is not a modifier key)
Args:
raw_demo (dict): json contents of demo file
field_extractor (FieldsExtractor): the fields extractor for this task
find_shortcuts (bool): whether to also find possible shortcuts
in the graph. If false, the graph will be a sequence.
Returns:
state_vertices (list[StateVertex])
"""
utterance = raw_demo['utterance']
if 'fields' in raw_demo:
fields = Fields(raw_demo['fields'])
else:
fields = field_extractor(utterance)
raw_states = raw_demo['states']
# Group BEFORE and AFTER
# Some AFTER are missing due to event propagation being stopped,
# in which case we also use BEFORE for AFTER
raw_state_pairs = []
current_before = None
for i, raw_state in enumerate(raw_states[1:]):
if raw_state['action']['type'] == 'scroll':
# Skip all scroll actions
continue
if raw_state['action']['timing'] == EpisodeGraph.BEFORE:
if current_before:
# Two consecutive BEFOREs
logging.warning('state %d is BEFORE without AFTER', i - 1)
raw_state_pairs.append((current_before, current_before))
current_before = raw_state
elif raw_state['action']['timing'] == EpisodeGraph.AFTER:
if not current_before:
# Two consecutive AFTERs
logging.warning('state %d is AFTER without BEFORE', i)
current_before = raw_state
raw_state_pairs.append((current_before, raw_state))
current_before = None
if current_before:
# Lingering BEFORE at the end
logging.warning('state %d is BEFORE without AFTER', i - 1)
raw_state_pairs.append((current_before, current_before))
if self._logfile:
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
# print >> self._logfile, '#' * 10, 'PAIRS'
self._logfile.write('Utterance: {}'.format(utterance))
self._logfile.write('Fields: {}'.format(fields))
self._logfile.write('#' * 10 + 'PAIRS')
for i, (s1, s2) in enumerate(raw_state_pairs):
self._logfile.write('@' + str(i) + ':' + str(s1['action']) +
str(s2['action']))
chunks = self._chunk_events(raw_state_pairs, utterance, fields)
chunks = self._collapse_type_actions(chunks)
if self._logfile:
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
# print >> self._logfile, '#' * 10, 'CHUNKS'
self._logfile.write('Utterance: {}'.format(utterance))
self._logfile.write('Fields:{}'.format(fields))
self._logfile.write('#' * 10 + 'CHUNKS')
for i, chunk in enumerate(chunks):
# print >> self._logfile, '@', i, ':', chunk
self._logfile.write('@' + str(i) + ':' + str(chunk))
# Create base vertices
state_vertices = []
for chunk in chunks:
start, end = len(state_vertices), len(state_vertices) + 1
if not chunk.target:
# Probably clicking/dragging on the instruction box
continue
if chunk.action == 'click':
action = MiniWoBElementClick(chunk.target)
if chunk.target.is_leaf:
action_edge = ActionEdge(action, start, end)
else:
action_edge = DummyActionEdge(chunk, start, end, 'nonleaf')
elif chunk.action == 'type':
action = MiniWoBFocusAndType(chunk.target, chunk.args)
if chunk.target.is_leaf:
action_edge = ActionEdge(action, start, end)
else:
action_edge = DummyActionEdge(chunk, start, end, 'nonleaf')
else:
action_edge = DummyActionEdge(chunk, start, end, 'unknown')
# If we don't plan to find shortcuts, we cannot have dummy edges
if not find_shortcuts and isinstance(action_edge, DummyActionEdge):
continue
state_vertex = StateVertex(chunk.state, [action_edge])
state_vertices.append(state_vertex)
if self._logfile:
# print >> self._logfile, '#' * 10, 'GRAPH'
self._logfile.write('#' * 10 + 'GRAPH')
for i, v in enumerate(state_vertices):
self._logfile.write('@' + str(i) + ':' + str(v.action_edges))
self._logfile.write(v.state.dom.visualize())
if find_shortcuts:
if self._logfile:
# print >> self._logfile, '#' * 10, 'SHORTCUTS'
self._logfile.write('#' * 10 + 'SHORTCUTS')
self._find_shortcuts(state_vertices)
# Remove dummy edges
for i, state_vertex in enumerate(state_vertices):
state_vertex.action_edges[:] = [
e for e in state_vertex.action_edges
if not isinstance(e, DummyActionEdge)
]
# To prevent empty states, add skip edges to the next state
if not state_vertex.action_edges:
state_vertex.action_edges.append(ActionEdge(None, i, i + 1))
if self._logfile:
# print >> self._logfile, '#' * 10, 'FINAL'
# print >> self._logfile, 'Utterance:', utterance
# print >> self._logfile, 'Fields:', fields
self._logfile.write('#' * 10 + 'FINAL')
self._logfile.write('Utterance:{}'.format(utterance))
self._logfile.write('Fields:{}'.format(fields))
for i, v in enumerate(state_vertices):
# print >> self._logfile, '@', i, ':', v.action_edges
self._logfile.write('@' + str(i) + ':' + str(v.action_edges))
return state_vertices
def _parse_raw_demo_original(self, raw_demo, field_extractor):
"""Takes the raw demo and spits out the relevant states.
Algorithm: Look at mousedown / keypress events
Args:
raw_demo (dict): json contents of demo file
field_extractor (FieldsExtractor): the fields extractor for this task
Returns:
state_vertices (list[StateVertex])
"""
# Filter out only for keypresses and mousedowns (BEFORE)
utterance = raw_demo['utterance']
if 'fields' in raw_demo:
fields = Fields(raw_demo['fields'])
else:
fields = field_extractor(utterance)
raw_states = raw_demo["states"]
state_vertices = []
actions = []
vertex_number = 0
for i, raw_state in enumerate(raw_states[1:]):
raw_action = raw_state["action"]
if raw_action["timing"] == EpisodeGraph.BEFORE:
if raw_action["type"] == "mousedown":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
click = MiniWoBElementClick(target)
state_vertex = StateVertex(
miniwob_state,
[ActionEdge(click, vertex_number, vertex_number + 1)])
state_vertices.append(state_vertex)
vertex_number += 1
elif raw_action["type"] == "keypress":
miniwob_state = MiniWoBState(utterance, fields,
raw_state['dom'])
char = chr(raw_action["keyCode"])
target = self._target(miniwob_state.dom_elements)
if not target: # target = yellow instruction box
continue
type_action = MiniWoBFocusAndType(target, char)
state_vertex = StateVertex(miniwob_state, [
ActionEdge(type_action, vertex_number,
vertex_number + 1)
])
state_vertices.append(state_vertex)
vertex_number += 1
# Collapse consecutive FocusAndTypes into one
for i, vertex in enumerate(state_vertices):
curr_action = vertex.action_edges[0].action
if not isinstance(curr_action, MiniWoBFocusAndType):
continue
aggregated_text = curr_action.text
while i + 1 < len(state_vertices):
next_action = state_vertices[i + 1].action_edges[0].action
if not isinstance(next_action, MiniWoBFocusAndType) or \
curr_action.element != next_action.element:
break
aggregated_text += next_action.text
del next_action
del state_vertices[i + 1]
vertex.action_edges[0] = ActionEdge(
MiniWoBFocusAndType(curr_action.element, aggregated_text), i,
i + 1)
# Collapse Click then FocusAndType into just FocusAndType
collapsed_state_vertices = []
for index in range(len(state_vertices) - 1):
curr_action = state_vertices[index].action_edges[0].action
next_action = state_vertices[index + 1].action_edges[0].action
if not(isinstance(curr_action, MiniWoBElementClick) and \
isinstance(next_action, MiniWoBFocusAndType) and \
curr_action.element == next_action.element):
collapsed_state_vertices.append(state_vertices[index])
collapsed_state_vertices.append(state_vertices[-1])
# Correct the edge indices
for i, state_vertex in enumerate(collapsed_state_vertices):
state_vertex.action_edges[0] = ActionEdge(
state_vertex.action_edges[0].action, i, i + 1)
return collapsed_state_vertices