def __init__(self, opt): self.opt = opt self.datatype = opt.get('datatype') self.datafile = build_data(self.opt) self._setup_data() self.length_datafile = self.datafile + ".length" self.training = self.datatype.startswith('train') self._load_lens()
def __init__(self, opt): self.opt = opt self.datatype = opt.get('datatype') self.datapath = build_data(self.opt) self.length_datafile = os.path.join(self.datapath, 'data_length') self.datafile = os.path.join(self.datapath, 'data') self.training = self.datatype.startswith('train') self._load_lens() self._setup_data()
def __init__(self, opt): self.opt = opt self.datatype = opt.get('datatype') self.datafile = build_data(self.opt) self.data_gen = self._data_generator(self.datafile) self.length_datafile = self.datafile + ".length" self.num_epochs = self.opt.get('num_epochs', 0) self.training = self.datatype.startswith('train') self._load_lens()
def __init__(self, opt): self.opt = opt self.datatype = opt.get('datatype') self.datapath = build_data(self.opt) self.length_datafile = os.path.join(self.datapath, 'data_length') self.char_index_file = os.path.join(self.datapath, 'char_index') self.datafile = os.path.join(self.datapath, 'data') self.training = self.datatype.startswith('train') self.ordered = ('ordered' in self.datatype or ('stream' in self.datatype and not opt.get('shuffle'))) self._load_lens()
# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Generates a pytorch data file from the training data; for use in the PytorchDataTeacher. Note that with our given implementation of batch act, episodes are compressed such that each episode is one example for a model. One can set the `--context-len` flag to specify how many past utterances are used in a flattened episode """ from parlai.scripts.build_pytorch_data import setup_args, build_data if __name__ == '__main__': opt = setup_args().parse_args() build_data(opt)
# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Generates a pytorch data file from the training data; for use in the PytorchDataTeacher. Note that with our given implementation of batch act, episodes are compressed such that each episode is one example for a model. One can set the `--context-len` flag to specify how many past utterances are used in a flattened episode """ from parlai.scripts.build_pytorch_data import setup_args, build_data if __name__ == '__main__': parser = setup_args() build_data(parser)