def _generate_no_verb_actions(ui_object_list):
  """Generates action based on no verb rule, action_type should be CLICK.

  action = [object_description_str]

  Args:
    ui_object_list: list of ui objects

  Returns:
    A list of common.Action instances.
  """
  action_list = []
  action_rule = common.ActionRules.NO_VERB_RULE

  for object_index, ui_object in enumerate(ui_object_list):
    if _valid_clickable_object_with_name(ui_object):
      action_type = common.ActionTypes.CLICK
      verb_str = ''
      obj_desc_str = _truncate_name(ui_object.obj_name,
                                    config.MAX_OBJ_NAME_WORD_NUM)
      action = common.Action(
          verb_str=verb_str,
          obj_desc_str=obj_desc_str,
          input_content_str=config.LABEL_DEFAULT_VALUE_STRING,
          action_type=action_type,
          action_rule=action_rule,
          target_obj_idx=object_index)
      action_list.append(action)

  for action_element in action_list:
    _fill_action_info(action_element)
  return action_list
def _generate_swipe_actions(ui_object_list):
  """Generates swipe actions and targets.

  Rule:
    action = [verb, swipe_direction*, swipe_object_str*]
    target = ui_object's index in ui_object_list
    (* means optional)
  Args:
    ui_object_list:

  Returns:
    A list of common.Action instances.
  """
  action_list = []
  action_rule = common.ActionRules.SWIPE_TO_OBJECT_RULE

  for object_index, ui_object in enumerate(ui_object_list):
    if _valid_object_with_name(ui_object):
      (verb_str,
       action_type) = _get_verb_str_action_type('swipe', ui_object.obj_type)
      obj_desc_str = _get_obj_desc_str(action_rule, ui_object)
      action = common.Action(
          verb_str=verb_str,
          obj_desc_str=obj_desc_str,
          input_content_str=config.LABEL_DEFAULT_VALUE_STRING,
          action_type=action_type,
          action_rule=action_rule,
          target_obj_idx=object_index)
      action_list.append(action)

  for action_element in action_list:
    _fill_action_info(action_element)
  return action_list
예제 #3
0
def _generate_relative_location_rule_action(target_object, target_object_id,
                                            neighbor_object,
                                            context_direction_str):
    """Generates action based on neighbor context rule.

  action = [verb, string_content*, target_object_type, context_direction_str,
            neighbor_name_str, neighbor_type_str]
  A is target_object, B is neighbor_object.

  Args:
    target_object: ui object.
    target_object_id: target ui object id.
    neighbor_object: target ui object's neighbor object
    context_direction_str: a string that describs target object and neighbor
      object's context relation.

  Returns:
    A list of common.Action instances.
  """
    input_content_list = []
    action_result_list = []
    action_rule = common.ActionRules.NEIGHBOR_CONTEXT_RULE

    if _valid_clickable_object(target_object):
        (verb_str,
         action_type) = _get_verb_str_action_type('click',
                                                  target_object.obj_type)
        input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
    elif _valid_typable_object_with_name(target_object):
        (verb_str,
         action_type) = _get_verb_str_action_type('input',
                                                  target_object.obj_type)
        input_content_list = [
            _generate_string_seq()
            for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
        ]
    obj_desc_str = _get_obj_desc_str(
        action_rule,
        neighbor_object,
        context_direction_str=context_direction_str,
        target_ui_object=target_object)

    for input_content_str in input_content_list:
        action = common.Action(verb_str=verb_str,
                               obj_desc_str=obj_desc_str,
                               input_content_str=input_content_str,
                               action_type=action_type,
                               action_rule=action_rule,
                               target_obj_idx=target_object_id)
        action_result_list.append(action)
    for action_element in action_result_list:
        _fill_action_info(action_element)
    return action_result_list
def _generate_single_object_rule_action(ui_object, target_object_id, platform):
  """Generates action based on single object rule.

  action = [verb, string_content*, object_description_str, object_type_str]

  Args:
    ui_object: ui object.
    target_object_id: target ui object id.
    platform: platform of UI objects.

  Returns:
    A list of common.Action instances.
  """
  input_content_list = []
  action_result_list = []
  action_rule = common.ActionRules.SINGLE_OBJECT_RULE

  if _valid_clickable_object_with_name(ui_object, platform):  # for CLICK
    (verb_str, action_type) = _get_verb_str_action_type('click',
                                                        ui_object.obj_type,
                                                        platform)
    input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
  elif _valid_typable_object_with_name(ui_object, platform):  # for INPUT
    (verb_str, action_type) = _get_verb_str_action_type('input',
                                                        ui_object.obj_type,
                                                        platform)
    input_content_list = [
        _generate_string_seq()
        for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
    ]
  else:
    return action_result_list
  obj_desc_str = _get_obj_desc_str(action_rule, ui_object, platform=platform)

  for input_content_str in input_content_list:
    action = common.Action(
        verb_str=verb_str,
        obj_desc_str=obj_desc_str,
        input_content_str=input_content_str,
        action_type=action_type,
        action_rule=action_rule,
        target_obj_idx=target_object_id)
    action_result_list.append(action)
  for action_element in action_result_list:
    _fill_action_info(action_element)
  return action_result_list
def _generate_absolute_location_rule_action(ui_object, target_object_id,
                                            grid_direction_str):
  """Generates action based on grid context rule.

  action = [verb, string_content*, object_type, grid_direction_str]

  Args:
    ui_object: ui object.
    target_object_id: target ui object id.
    grid_direction_str: a string that describs grid dicretion.

  Returns:
    A list of common.Action instances.
  """
  action_rule = common.ActionRules.GRID_CONTEXT_RULE
  action_result_list = []
  input_content_list = []
  if _valid_clickable_object(ui_object):
    (verb_str, action_type) = _get_verb_str_action_type('click',
                                                        ui_object.obj_type)
    input_content_list = [config.LABEL_DEFAULT_VALUE_STRING]
  elif _valid_typable_object_with_name(ui_object):
    (verb_str, action_type) = _get_verb_str_action_type('input',
                                                        ui_object.obj_type)
    input_content_list = [
        _generate_string_seq()
        for _ in range(config.INPUT_ACTION_UPSAMPLE_RATIO)
    ]
  obj_desc_str = _get_obj_desc_str(
      action_rule, ui_object, context_direction_str=grid_direction_str)
  for input_content_str in input_content_list:
    action = common.Action(
        verb_str=verb_str,
        obj_desc_str=obj_desc_str,
        input_content_str=input_content_str,
        action_type=action_type,
        action_rule=action_rule,
        target_obj_idx=target_object_id)
    action_result_list.append(action)
  for action_element in action_result_list:
    _fill_action_info(action_element)
  return action_result_list