def _generate_movielens_examples(cls,
data_dir,
generated_examples_dir,
train_filename,
test_filename,
vocab_filename,
meta_filename,
min_timeline_length=3,
max_context_length=10):
"""Generate movielens examples, and returns a dict contains meta."""
train_file = os.path.join(generated_examples_dir, train_filename)
test_file = os.path.join(generated_examples_dir, test_filename)
meta_file = os.path.join(generated_examples_dir, meta_filename)
# Create dataset and meta, only if they are not existed.
if not all([os.path.exists(f) for f in (train_file, test_file, meta_file)]):
stats = _gen.generate_datasets(
data_dir,
output_dir=generated_examples_dir,
min_timeline_length=min_timeline_length,
max_context_length=max_context_length,
build_movie_vocab=True,
train_filename=train_filename,
test_filename=test_filename,
vocab_filename=vocab_filename,
)
file_util.write_json_file(meta_file, stats)
meta = file_util.load_json_file(meta_file)
return meta
def from_squad(cls,
filename,
model_spec,
is_training=True,
version_2_with_negative=False,
cache_dir=None):
"""Loads data in SQuAD format and preproecess text according to `model_spec`.
Args:
filename: Name of the file.
model_spec: Specification for the model.
is_training: Whether the loaded data is for training or not.
version_2_with_negative: Whether it's SQuAD 2.0 format.
cache_dir: The cache directory to save preprocessed data. If None,
generates a temporary directory to cache preprocessed data.
Returns:
QuestionAnswerDataLoader object.
"""
model_spec = ms.get(model_spec)
file_base_name = os.path.basename(filename)
is_cached, tfrecord_file, meta_data_file, _ = _get_cache_info(
cache_dir, file_base_name, model_spec, is_training)
# If cached, directly loads data from cache directory.
if is_cached and is_training:
dataset, meta_data = _load(tfrecord_file, meta_data_file,
model_spec, is_training)
return QuestionAnswerDataLoader(
dataset=dataset,
size=meta_data['size'],
version_2_with_negative=meta_data['version_2_with_negative'],
examples=[],
features=[],
squad_file=filename)
meta_data, examples, features = cls._generate_tf_record_from_squad_file(
filename, model_spec, tfrecord_file, is_training,
version_2_with_negative)
file_util.write_json_file(meta_data_file, meta_data)
dataset, meta_data = _load(tfrecord_file, meta_data_file, model_spec,
is_training)
return QuestionAnswerDataLoader(dataset, meta_data['size'],
meta_data['version_2_with_negative'],
examples, features, filename)
def _save_data(cls, examples, model_spec, label_names, tfrecord_file,
meta_data_file, vocab_file, is_training):
"""Saves preprocessed data and other assets into files."""
# If needed, generates and saves vocabulary in vocab_file=None,
if model_spec.need_gen_vocab and is_training:
model_spec.gen_vocab(examples)
model_spec.save_vocab(vocab_file)
# Converts examples into preprocessed features and saves in tfrecord_file.
model_spec.convert_examples_to_features(examples, tfrecord_file,
label_names)
# Generates and saves meta data in meta_data_file.
meta_data = {
'size': len(examples),
'num_classes': len(label_names),
'index_to_label': label_names
}
file_util.write_json_file(meta_data_file, meta_data)
def generate_movielens_dataset(
cls,
data_dir,
generated_examples_dir=None,
train_filename='train_movielens_1m.tfrecord',
test_filename='test_movielens_1m.tfrecord',
vocab_filename='movie_vocab.json',
meta_filename='meta.json',
min_timeline_length=3,
max_context_length=10,
max_context_movie_genre_length=10,
min_rating=None,
train_data_fraction=0.9,
build_vocabs=True,
):
"""Generate movielens dataset, and returns a dict contains meta.
Args:
data_dir: str, path to dataset containing (unzipped) text data.
generated_examples_dir: str, path to generate preprocessed examples.
(default: same as data_dir)
train_filename: str, generated file name for training data.
test_filename: str, generated file name for test data.
vocab_filename: str, generated file name for vocab data.
meta_filename: str, generated file name for meta data.
min_timeline_length: int, min timeline length to split train/eval set.
max_context_length: int, max context length as one input.
max_context_movie_genre_length: int, max context length of movie genre as
one input.
min_rating: int or None, include examples with min rating.
train_data_fraction: float, percentage of training data [0.0, 1.0].
build_vocabs: boolean, whether to build vocabs.
Returns:
Dict, metadata for the movielens dataset. Containing keys:
`train_file`, `train_size`, `test_file`, `test_size`, vocab_file`,
`vocab_size`, etc.
"""
if not generated_examples_dir:
# By default, set generated examples dir to data_dir
generated_examples_dir = data_dir
train_file = os.path.join(generated_examples_dir, train_filename)
test_file = os.path.join(generated_examples_dir, test_filename)
meta_file = os.path.join(generated_examples_dir, meta_filename)
# Create dataset and meta, only if they are not existed.
if not all(
[os.path.exists(f) for f in (train_file, test_file, meta_file)]):
stats = _gen.generate_datasets(
data_dir,
output_dir=generated_examples_dir,
min_timeline_length=min_timeline_length,
max_context_length=max_context_length,
max_context_movie_genre_length=max_context_movie_genre_length,
min_rating=min_rating,
build_vocabs=build_vocabs,
train_data_fraction=train_data_fraction,
train_filename=train_filename,
test_filename=test_filename,
vocab_filename=vocab_filename,
)
file_util.write_json_file(meta_file, stats)
meta = file_util.load_json_file(meta_file)
return meta