def _generate_no_verb_actions(ui_object_list):
"""Generates action based on no verb rule, action_type should be CLICK.
action = [object_description_str]
Args:
ui_object_list: list of ui objects
Returns:
A list of common.Action instances.
"""
action_list = []
action_rule = common.ActionRules.NO_VERB_RULE
for object_index, ui_object in enumerate(ui_object_list):
if _valid_clickable_object_with_name(ui_object):
action_type = common.ActionTypes.CLICK
verb_str = ''
obj_desc_str = _truncate_name(ui_object.obj_name,
config.MAX_OBJ_NAME_WORD_NUM)
action = common.Action(
verb_str=verb_str,
obj_desc_str=obj_desc_str,
input_content_str=config.LABEL_DEFAULT_VALUE_STRING,
action_type=action_type,
action_rule=action_rule,
target_obj_idx=object_index)
action_list.append(action)
for action_element in action_list:
_fill_action_info(action_element)
return action_list
def _generate_swipe_actions(ui_object_list):
"""Generates swipe actions and targets.
Rule:
action = [verb, swipe_direction*, swipe_object_str*]
target = ui_object's index in ui_object_list
(* means optional)
Args:
ui_object_list:
Returns:
A list of common.Action instances.
"""
action_list = []
action_rule = common.ActionRules.SWIPE_TO_OBJECT_RULE
for object_index, ui_object in enumerate(ui_object_list):
if _valid_object_with_name(ui_object):
(verb_str,
action_type) = _get_verb_str_action_type('swipe', ui_object.obj_type)
obj_desc_str = _get_obj_desc_str(action_rule, ui_object)
action = common.Action(
verb_str=verb_str,
obj_desc_str=obj_desc_str,
input_content_str=config.LABEL_DEFAULT_VALUE_STRING,
action_type=action_type,
action_rule=action_rule,
target_obj_idx=object_index)
action_list.append(action)
for action_element in action_list:
_fill_action_info(action_element)
return action_list
def _generate_relative_location_rule_action(target_object, target_object_id,
neighbor_object,
context_direction_str):
"""Generates action based on neighbor context rule.
action = [verb, string_content*, target_object_type, context_direction_str,
neighbor_name_str, neighbor_type_str]
A is target_object, B is neighbor_object.
Args:
target_object: ui object.
target_object_id: target ui object id.
neighbor_object: target ui object's neighbor object
context_direction_str: a string that describs target object and neighbor
object's context relation.
Returns:
A list of common.Action instances.
"""
input_content_list = []
action_result_list = []
action_rule = common.ActionRules.NEIGHBOR_CONTEXT_RULE
if _valid_clickable_object(target_object):
(verb_str,
action_type) = _get_verb_str_action_type('click',
target_object.obj_type)
input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
elif _valid_typable_object_with_name(target_object):
(verb_str,
action_type) = _get_verb_str_action_type('input',
target_object.obj_type)
input_content_list = [
_generate_string_seq()
for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
]
obj_desc_str = _get_obj_desc_str(
action_rule,
neighbor_object,
context_direction_str=context_direction_str,
target_ui_object=target_object)
for input_content_str in input_content_list:
action = common.Action(verb_str=verb_str,
obj_desc_str=obj_desc_str,
input_content_str=input_content_str,
action_type=action_type,
action_rule=action_rule,
target_obj_idx=target_object_id)
action_result_list.append(action)
for action_element in action_result_list:
_fill_action_info(action_element)
return action_result_list
def _generate_single_object_rule_action(ui_object, target_object_id, platform):
"""Generates action based on single object rule.
action = [verb, string_content*, object_description_str, object_type_str]
Args:
ui_object: ui object.
target_object_id: target ui object id.
platform: platform of UI objects.
Returns:
A list of common.Action instances.
"""
input_content_list = []
action_result_list = []
action_rule = common.ActionRules.SINGLE_OBJECT_RULE
if _valid_clickable_object_with_name(ui_object, platform): # for CLICK
(verb_str, action_type) = _get_verb_str_action_type('click',
ui_object.obj_type,
platform)
input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
elif _valid_typable_object_with_name(ui_object, platform): # for INPUT
(verb_str, action_type) = _get_verb_str_action_type('input',
ui_object.obj_type,
platform)
input_content_list = [
_generate_string_seq()
for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
]
else:
return action_result_list
obj_desc_str = _get_obj_desc_str(action_rule, ui_object, platform=platform)
for input_content_str in input_content_list:
action = common.Action(
verb_str=verb_str,
obj_desc_str=obj_desc_str,
input_content_str=input_content_str,
action_type=action_type,
action_rule=action_rule,
target_obj_idx=target_object_id)
action_result_list.append(action)
for action_element in action_result_list:
_fill_action_info(action_element)
return action_result_list
def _generate_absolute_location_rule_action(ui_object, target_object_id,
grid_direction_str):
"""Generates action based on grid context rule.
action = [verb, string_content*, object_type, grid_direction_str]
Args:
ui_object: ui object.
target_object_id: target ui object id.
grid_direction_str: a string that describs grid dicretion.
Returns:
A list of common.Action instances.
"""
action_rule = common.ActionRules.GRID_CONTEXT_RULE
action_result_list = []
input_content_list = []
if _valid_clickable_object(ui_object):
(verb_str, action_type) = _get_verb_str_action_type('click',
ui_object.obj_type)
input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
elif _valid_typable_object_with_name(ui_object):
(verb_str, action_type) = _get_verb_str_action_type('input',
ui_object.obj_type)
input_content_list = [
_generate_string_seq()
for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
]
obj_desc_str = _get_obj_desc_str(
action_rule, ui_object, context_direction_str=grid_direction_str)
for input_content_str in input_content_list:
action = common.Action(
verb_str=verb_str,
obj_desc_str=obj_desc_str,
input_content_str=input_content_str,
action_type=action_type,
action_rule=action_rule,
target_obj_idx=target_object_id)
action_result_list.append(action)
for action_element in action_result_list:
_fill_action_info(action_element)
return action_result_list
