def _get_view_hierachy(file_path):
"""Gets leaf nodes view hierarchy lists.
Args:
file_path: The full path of an input xml/json file.
Returns:
A ViewHierarchy object.
Raises:
ValueError: unsupported file format.
"""
with gfile.GFile(file_path, 'r') as f:
data = f.read()
_, file_extension = os.path.splitext(file_path)
if file_extension == '.xml':
vh = view_hierarchy.ViewHierarchy(screen_width=config.SCREEN_WIDTH,
screen_height=config.SCREEN_HEIGHT)
vh.load_xml(data)
elif file_extension == '.json':
vh = view_hierarchy.ViewHierarchy(
screen_width=config.RICO_SCREEN_WIDTH,
screen_height=config.RICO_SCREEN_HEIGHT)
vh.load_json(data)
else:
raise ValueError('unsupported file format %s' % file_extension)
return vh
def get_feature_dict(screen_info_proto, padding_shape=None, lower_case=False):
"""Gets screen feature dictionary from screen_info protobuf.
Args:
screen_info_proto: protobuf defined in word2act/proto/rehearsal.proto.
Contains screenshot and xml
padding_shape: The shape of padding size for final feature list. shape =
(max_object_num, max_word_num, max_word_length) If the shape is not given,
then returns the original list without padding.
lower_case: lower case all the ui texts.
Returns:
A feature dictionary. If padding_shape is not None, all values of the
dictionary are padded. The shape after padding is shown as 'shape = ...'.
Otherwise, shapes of values are not a fixed value.
screenshot: numpy array of screen_info_proto.screenshot
'ui_obj_str_seq': uiobject's name/content_descriotion/resource_id, numpy
array of strings.
'ui_obj_word_id_seq': encoded word sequence, np int array, shape =
(max_object_num, max_word_num)
'ui_obj_char_id_seq': encoded char sequence, np int array, shape =
(max_object_num, max_word_num, max_word_length)
'ui_obj_type_seq': type sequence, np int array, shape = (max_object_num,)
'ui_obj_clickable_seq': clickable sequence, np int array, shape =
(max_object_num,)
'ui_obj_cord_x_seq': x cordinate sequence, np int array, shape =
(max_object_num*2,)
'ui_obj_cord_y_seq': y cordinate sequence, np int array, shape =
(max_object_num*2,)
'ui_obj_v_distance': vertical relation matrix, np float array,
shape = (max_object_num, max_object_num)
'ui_obj_h_distance': horizontal relation matrix, np float array, shape =
(max_object_num, max_object_num)
'ui_obj_dom_distance': dom relation matrix, np int array, shape =
(max_object_num, max_object_num)
'ui_obj_dom_location_seq': dom index from tree traversal, np int array,
shape = (max_object_num*3,)
"""
screenshot = Image.open(io.BytesIO(screen_info_proto.screenshot.content))
screenshot = np.asarray(screenshot, np.float32)
vh = view_hierarchy.ViewHierarchy()
vh.load_xml(screen_info_proto.view_hierarchy.xml.encode('utf-8'))
view_hierarchy_leaf_nodes = vh.get_leaf_nodes()
ui_object_features_dict = get_ui_objects_feature_dict(
view_hierarchy_leaf_nodes, padding_shape, lower_case)
ui_object_features_dict['screenshot'] = screenshot
return ui_object_features_dict
def update_info_from_screen(self, screen_info, dedup=False):
"""Updates action event attributes from screen_info.
Updates coordinates_x(y)_pixel and object_id from the screen_info proto.
Args:
screen_info: ScreenInfo protobuf
dedup: whether dedup the UI objs with same text or content desc.
Raises:
ValueError when fail to find object id.
"""
self.update_norm_coordinates(
(config.SCREEN_WIDTH, config.SCREEN_HEIGHT))
vh = view_hierarchy.ViewHierarchy()
vh.load_xml(screen_info.view_hierarchy.xml.encode('utf-8'))
if dedup:
vh.dedup(
(self.coordinates_x_pixel[0], self.coordinates_y_pixel[0]))
self.leaf_nodes = vh.get_leaf_nodes()
ui_object_list = vh.get_ui_objects()
self._update_object_id(ui_object_list)