def preprocess_mapper(features, params, lookup_table, vocab, mode): """Model-specific preprocessing of features from the dataset.""" # Set input type. features["input_type"] = tf.constant(datasets.DatasetTypes.REFERENCE) if mode != tf.estimator.ModeKeys.PREDICT: # Select random caption. captions = tf.io.parse_tensor(features["captions"], tf.string) num_captions = tensor_utils.shape(captions, 0) rid = tf.random.uniform([], maxval=num_captions, dtype=tf.int32) caption = text_utils.build_text_inputs(text=captions[rid], length=params["caption_length"], lookup_table=lookup_table, segment_id=0, start_token=vocab.CLS, end_token=vocab.SEP) assert isinstance(caption, text_utils.TextInputs) features["token_inputs"] = text_utils.TextInputs( token_ids=caption.token_ids[:-1], mask=caption.mask[:-1], segment_ids=caption.segment_ids[:-1], positions=caption.positions[:-1]) features["token_outputs"] = text_utils.TextOutputs( token_ids=caption.token_ids[1:], mask=caption.mask[1:]) if params.get("conditional_decoding"): random_span = text_utils.get_random_span( text=captions[rid], p=params["span_sample_p"], max_span_len=params["span_length"]) features["condition_inputs"] = text_utils.build_text_inputs( text=random_span, length=params["condition_length"], lookup_table=lookup_table, segment_id=1, start_token=vocab.ANS) features["object_features"] = image_utils.parse_object_features( features["object_features"], features["object_positions"], params) # Remove extra inputs. features = {f: features[f] for f in features if f in KEYS} # Add dummy inputs for standardization for multi-tasking. footprint = datasets.footprint(params) assert footprint for k, v in footprint.items(): if k not in features: features[k] = v return features
def preprocess_mapper(features, params, lookup_table, vocab): """Model-specific preprocessing of features from the dataset.""" # Set input type. features["input_type"] = tf.constant(datasets.DatasetTypes.VQA) # Fix question id. features["question_id"] = tf.ensure_shape(features["question_id"], []) features["question_inputs"] = text_utils.build_text_inputs( text=features["question"], length=params["question_length"], lookup_table=lookup_table, segment_id=0) answer = text_utils.build_text_inputs( text=features["answer"], length=params["answer_length"], lookup_table=lookup_table, segment_id=1, start_token=vocab.ANS) assert isinstance(answer, text_utils.TextInputs) features["answer_inputs"] = answer features["answer_outputs"] = text_utils.TextOutputs( token_ids=answer.token_ids[1:], mask=answer.mask[1:]) features["object_features"] = image_utils.parse_object_features( features["object_features"], features["object_positions"], params) if params.get("conditional_decoding"): features["condition_inputs"] = text_utils.build_planner_inputs( question=features["question"], answer=features["answer"], length=params["condition_length"], lookup_table=lookup_table) # Remove extra inputs. features = {f: features[f] for f in features if f in KEYS} # Add dummy inputs for standardization for multi-tasking. for k, v in datasets.footprint(params).items(): if k not in features: features[k] = v return features
def preprocess_mapper(features, params, lookup_table, vocab): """Model-specific preprocessing of features from the dataset.""" # Set input type. features["input_type"] = tf.constant(datasets.DatasetTypes.GUIDED) caption = text_utils.build_text_inputs(text=features["caption"], length=params["caption_length"], lookup_table=lookup_table, segment_id=0, start_token=vocab.CLS, end_token=vocab.SEP) assert isinstance(caption, text_utils.TextInputs) features["token_inputs"] = text_utils.TextInputs( token_ids=caption.token_ids[:-1], mask=caption.mask[:-1], segment_ids=caption.segment_ids[:-1], positions=caption.positions[:-1]) features["token_outputs"] = text_utils.TextOutputs( token_ids=caption.token_ids[1:], mask=caption.mask[1:]) features["condition_inputs"] = text_utils.build_planner_inputs( question=features["question"], answer=features["answer"], length=params["condition_length"], lookup_table=lookup_table) features["object_features"] = image_utils.parse_object_features( features["object_features"], features["object_positions"], params) # Remove extra inputs. features = {f: features[f] for f in features if f in KEYS} # Add dummy inputs for standardization for multi-tasking. for k, v in datasets.footprint(params).items(): if k not in features: features[k] = v return features
def preprocess_mapper(raw_text, params, lookup_table, vocab): """Model-specific preprocessing of features from the dataset.""" features = dict(input_type=datasets.DatasetTypes.GUIDED) splits = tf.strings.split([raw_text], "\t").values question, answer, text = splits[1], splits[2], splits[3] text = text_utils.build_text_inputs(text=text, length=params["caption_length"], lookup_table=lookup_table, segment_id=0, start_token=vocab.CLS, end_token=vocab.SEP) assert isinstance(text, text_utils.TextInputs) features["token_inputs"] = text_utils.TextInputs( token_ids=text.token_ids[:-1], mask=text.mask[:-1], segment_ids=text.segment_ids[:-1], positions=text.positions[:-1]) features["token_outputs"] = text_utils.TextOutputs( token_ids=text.token_ids[1:], mask=text.mask[1:]) if params.get("conditional_decoding"): features["condition_inputs"] = text_utils.build_planner_inputs( question=question, answer=answer, length=params["condition_length"], lookup_table=lookup_table) # Add dummy inputs for standardization for multi-tasking. for k, v in datasets.footprint(params).items(): if k not in features: features[k] = v return features