def __init__(self, opt):
self.opt = opt
self.datatype = opt.get('datatype')
self.datafile = build_data(self.opt)
self._setup_data()
self.length_datafile = self.datafile + ".length"
self.training = self.datatype.startswith('train')
self._load_lens()
def __init__(self, opt):
self.opt = opt
self.datatype = opt.get('datatype')
self.datapath = build_data(self.opt)
self.length_datafile = os.path.join(self.datapath, 'data_length')
self.datafile = os.path.join(self.datapath, 'data')
self.training = self.datatype.startswith('train')
self._load_lens()
self._setup_data()
def __init__(self, opt):
self.opt = opt
self.datatype = opt.get('datatype')
self.datafile = build_data(self.opt)
self.data_gen = self._data_generator(self.datafile)
self.length_datafile = self.datafile + ".length"
self.num_epochs = self.opt.get('num_epochs', 0)
self.training = self.datatype.startswith('train')
self._load_lens()
def __init__(self, opt):
self.opt = opt
self.datatype = opt.get('datatype')
self.datapath = build_data(self.opt)
self.length_datafile = os.path.join(self.datapath, 'data_length')
self.char_index_file = os.path.join(self.datapath, 'char_index')
self.datafile = os.path.join(self.datapath, 'data')
self.training = self.datatype.startswith('train')
self.ordered = ('ordered' in self.datatype or
('stream' in self.datatype and not opt.get('shuffle')))
self._load_lens()
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
"""Generates a pytorch data file from the training data; for use in the
PytorchDataTeacher.
Note that with our given implementation of batch act, episodes are compressed
such that each episode is one example for a model.
One can set the `--context-len` flag to specify how many past utterances
are used in a flattened episode
"""
from parlai.scripts.build_pytorch_data import setup_args, build_data
if __name__ == '__main__':
opt = setup_args().parse_args()
build_data(opt)
# Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
"""Generates a pytorch data file from the training data; for use in the
PytorchDataTeacher.
Note that with our given implementation of batch act, episodes are compressed
such that each episode is one example for a model.
One can set the `--context-len` flag to specify how many past utterances
are used in a flattened episode
"""
from parlai.scripts.build_pytorch_data import setup_args, build_data
if __name__ == '__main__':
parser = setup_args()
build_data(parser)
