def _task_to_features_dict(task, do_stats): """Converts one task to features dict. Args: task: Task instance, the task to be converted. do_stats: whether do stats on this task. Set it to False for debug or test. Returns: features: dict of (string, np.array) which contains columns and values: features = { 'instruction_str': string of answer, np string array, shape = (1,) 'instruction_word_id_seq': word id sequence of question, np string array, shape = (word_num,) 'verb_id_seq': word id of verb, np int array, shape = (action_num,) 'obj_desc_position_seq': index of word id of object in answer, np int array, shape = (actions * 2,) 'input_str_position_seq': additional info of the action, shape = (action_num * 2,) 'raters_count_per_task': shape = (1,) 'agreement_count': shape = (1,) Raises: ValueError: raise error when fail to parse actions of tasks. """ answer = task['instruction'].lower() features = {} features['instruction_str'] = np.array([answer], dtype=np.string_) tokens, _ = string_utils.tokenize_to_ids(answer) features['instruction_word_id_seq'] = np.array(tokens, dtype=np.int64) verb_id_seq = [] verb_str_position_seq = [] obj_desc_position_seq = [] input_str_position_seq = [] for action in task['actions']: try: verb_id = common.ActionTypes[action.verb_type.upper().strip()] except KeyError: raise ValueError('Verb "%s" cannot be recognized.' % action.verb_type) if verb_id == common.ActionTypes.OTHERS: verb = answer[action.verb_start_pos:action.verb_end_pos].strip( ).lower() verb_id = common.VERB_ID_MAP.get(verb, common.ActionTypes.OTHERS) verb_id_seq.append(verb_id.value) verb_str_position_seq.extend( string_utils.get_token_pos_from_char_pos(answer, action.verb_start_pos, action.verb_end_pos)) if do_stats and task['agreement-count'] >= 2: distributions['longest_verb_str'][verb_str_position_seq[-1] - verb_str_position_seq[-2]] += 1 obj_desc_position_seq.extend( string_utils.get_token_pos_from_char_pos( answer, action.object_desc_start_pos, action.object_desc_end_pos)) if do_stats and task['agreement-count'] >= 2: distributions['longest_obj_desc'][obj_desc_position_seq[-1] - obj_desc_position_seq[-2]] += 1 if not (action.input_content_start_pos == 0 and action.input_content_end_pos == 0): input_str_position_seq.extend( string_utils.get_token_pos_from_char_pos( answer, action.input_content_start_pos, action.input_content_end_pos)) if do_stats and task['agreement-count'] >= 2: distributions['longest_input_str'][ input_str_position_seq[-1] - input_str_position_seq[-2]] += 1 else: input_str_position_seq.extend([config.LABEL_DEFAULT_VALUE_INT] * 2) features['verb_id_seq'] = np.array(verb_id_seq, dtype=np.int64) features['verb_str_position_seq'] = np.array(verb_str_position_seq, dtype=np.int64) features['obj_desc_position_seq'] = np.array(obj_desc_position_seq, dtype=np.int64) features['input_str_position_seq'] = np.array(input_str_position_seq, dtype=np.int64) features['agreement_count'] = np.array([task['agreement-count']], dtype=np.int64) if do_stats: distributions['step_num'][len(task['actions'])] += 1 distributions['longest_instruction'][len(tokens)] += 1 counters['total_verb_refs'] += len(verb_id_seq) counters['total_obj_refs'] += len(obj_desc_position_seq) / 2 counters['total_input_refs'] += ( (len(input_str_position_seq) - input_str_position_seq.count(config.LABEL_DEFAULT_VALUE_INT)) / 2) for verb in common.ActionTypes: if verb.value in verb_id_seq: counters['Instructions contain %s in verbs' % verb.name] += 1 if input_str_position_seq.count( config.LABEL_DEFAULT_VALUE_INT) != len(input_str_position_seq): counters['Instructions contain INPUT Content'] += 1 if ' and then ' in answer: chosen_examples['instruction_contains_and-then'].append(answer) if ' after ' in answer: chosen_examples['instruction_contains_after'].append(answer) if '. ' in answer: counters['instruction_contains_dot'] += 1 if ', ' in answer: counters['instruction_contains_comma'] += 1 return features
def _get_ui_object_attributes(view_hierarchy_leaf_nodes, lower_case=False): """Parses ui object informationn from a view hierachy leaf node list. Args: view_hierarchy_leaf_nodes: a list of view hierachy leaf nodes. lower_case: lower case all the ui texts. Returns: An un-padded attribute dictionary as follow: 'type_id_seq': numpy array of ui object types from view hierarchy. 'word_id_seq': numpy array of encoding for words in ui object. 'char_id_seq': numpy array of encoding for words in ui object. 'clickable_seq': numpy array of ui object clickable status. 'cord_x_seq': numpy array of ui object x coordination. 'cord_y_seq': numpy array of ui object y coordination. 'dom_location_seq': numpy array of ui object depth, pre-order-traversal index, post-order-traversal index. 'word_str_sequence': numpy array of ui object name strings. """ type_sequence = [] word_id_sequence = [] char_id_sequence = [] clickable_sequence = [] cord_x_sequence = [] cord_y_sequence = [] dom_location_sequence = [] obj_str_sequence = [] def _is_ascii(s): return all(ord(c) < 128 for c in s) for vh_node in view_hierarchy_leaf_nodes: ui_obj = vh_node.uiobject type_sequence.append(ui_obj.obj_type.value) cord_x_sequence.append(ui_obj.bounding_box.x1) cord_x_sequence.append(ui_obj.bounding_box.x2) cord_y_sequence.append(ui_obj.bounding_box.y1) cord_y_sequence.append(ui_obj.bounding_box.y2) clickable_sequence.append(ui_obj.clickable) dom_location_sequence.extend(ui_obj.dom_location) valid_words = [w for w in ui_obj.word_sequence if _is_ascii(w)] word_sequence = ' '.join(valid_words) if lower_case: word_sequence = word_sequence.lower() obj_str_sequence.append(word_sequence) word_ids, char_ids = string_utils.tokenize_to_ids(word_sequence) word_id_sequence.append(word_ids) char_id_sequence.append(char_ids) ui_feature = { 'type_id_seq': np.array(type_sequence), 'word_id_seq': np.array(word_id_sequence), 'clickable_seq': np.array(clickable_sequence), 'cord_x_seq': np.array(cord_x_sequence), 'cord_y_seq': np.array(cord_y_sequence), 'dom_location_seq': np.array(dom_location_sequence), 'obj_str_seq': np.array(obj_str_sequence, dtype=np.str), } return ui_feature
def get_synthetic_feature_dict(synthetic_action_list, max_word_num, unused_max_word_length, parse_consumed=False): """Get padded synthetic action feature dictionary. This dictionary contains all features related to the synthetic instructions. Args: synthetic_action_list: List of common.Action() instances max_word_num: max word number for padding unused_max_word_length: max word length for padding. parse_consumed: whether to parse consumed tag. Returns: a padded feature dictionary """ feature = { 'instruction_str': [], 'instruction_rule_id': [], 'instruction_word_id_seq': [], 'verb_id_seq': [], 'ui_target_id_seq': [], 'verb_str_position_seq': [], 'input_str_position_seq': [], 'obj_desc_position_seq': [], } if parse_consumed: feature['consumed_tag'] = [] feature['step_str_position_seq'] = [] for action in synthetic_action_list: if not action.is_valid(): continue action.convert_to_lower_case() word_id_seq, char_id_seq = string_utils.tokenize_to_ids( action.instruction_str) # skips the synthetic actions that have more than max_word_num tokens if len(word_id_seq) > max_word_num: tf.logging.info('[Dropped Long Synthetic Action]:%s', action.instruction_str) continue feature['instruction_str'].append(action.instruction_str) feature['instruction_rule_id'].append(action.action_rule.value) feature['instruction_word_id_seq'].append(word_id_seq) # Enable this when using word token if 'instruction_char_id_seq' in feature: feature['instruction_char_id_seq'].append(char_id_seq) feature['verb_id_seq'].append(action.action_type.value) feature['ui_target_id_seq'].append(action.target_obj_idx) feature['verb_str_position_seq'].extend( string_utils.get_token_pos_from_char_pos(action.instruction_str, action.verb_str_pos[0], action.verb_str_pos[1])) feature['obj_desc_position_seq'].extend( string_utils.get_token_pos_from_char_pos(action.instruction_str, action.obj_str_pos[0], action.obj_str_pos[1])) if action.has_valid_input(): feature['input_str_position_seq'].extend( string_utils.get_token_pos_from_char_pos(action.instruction_str, action.input_str_pos[0], action.input_str_pos[1])) else: feature['input_str_position_seq'].extend(action.input_str_pos) if parse_consumed: feature['consumed_tag'].append(int(action.is_consumed)) step_token_pos = string_utils.get_token_pos_from_char_pos( action.instruction_str, action.step_str_pos[0], action.step_str_pos[1]) feature['step_str_position_seq'].extend(step_token_pos) for key in feature: feature[key] = np.array(feature[key]) phrase_count = feature['instruction_str'].shape[0] feature_padding_info = { 'instruction_str': [phrase_count, np.string_, ''], 'instruction_rule_id': [(phrase_count), np.int64, 0], 'instruction_word_id_seq': [(phrase_count, max_word_num), np.int64, 0], 'verb_id_seq': [(phrase_count), np.int64, 0], 'ui_target_id_seq': [(phrase_count), np.int64, 0], 'verb_str_position_seq': [(phrase_count * 2), np.int64, 0], 'input_str_position_seq': [(phrase_count * 2), np.int64, 0], 'obj_desc_position_seq': [(phrase_count * 2), np.int64, 0], } if parse_consumed: feature_padding_info['consumed_tag'] = [(phrase_count), np.int64, 0] feature_padding_info['step_str_position_seq'] = [(phrase_count * 2), np.int64, 0] padding_shape, padding_type, padding_value = {}, {}, {} for key in feature_padding_info: shape, pad_type, value = feature_padding_info[key] padding_shape[key] = shape padding_type[key] = pad_type padding_value[key] = value padded_feature_dict = proto_utils.padding_dictionary(feature, padding_shape, padding_type, padding_value) return padded_feature_dict