class TestField(unittest.TestCase): def setUp(self): self.field = Field() def test_numericalize_and_preprocess(self): example = 1 result = self.field.numericalize(example) self.assertEqual(example, result) result = self.field.preprocess(example) self.assertEqual(example, result)
def test_from_json(self): json_obj = [{'source': [1, 2, 3], 'target': [4, 5, 6]}, {'source': [1, 2, 3], 'target': [4, 5, 6]}] file_path = '/path/to/json_file' import json # noqa: F401 open_patcher = patch('data.open') json_load_patcher = patch('json.load', return_value=json_obj) open_mock = open_patcher.start() json_load_mock = json_load_patcher.start() source = Field() target = Field() examples = Loader.from_json(file_path, {'source': source, 'target': target}) open_mock.assert_called_once_with(file_path) json_load_mock.assert_called_once_with(open_mock(file_path).__enter__()) self.assertEqual(len(examples['source']), 2) self.assertEqual(len(examples['target']), 2) self.assertListEqual(examples['source'].data, [[1, 2, 3], [1, 2, 3]]) self.assertListEqual(examples['target'].data, [[4, 5, 6], [4, 5, 6]]) open_patcher.stop() json_load_patcher.stop()
args.device = 'cuda' if torch.cuda.is_available() else 'cpu' args.mbr = True args.delete = { 'TOP', 'S1', '-NONE-', ',', ':', '``', "''", '.', '?', '!', '' } args.equal = {'ADVP': 'PRT'} args.train = os.path.join(args.data, 'train.pid') args.dev = os.path.join(args.data, 'dev.pid') args.test = os.path.join(args.data, 'test.pid') print("Building the fields") WORD = Field('words', pad='<pad>', unk='<unk>', bos='<bos>', eos='<eos>', lower=True) TAG = Field('tags', bos="<bos>", eos='<eos>') TREE = RawField('trees') global CHART CHART = ChartField('charts') transform = Tree(WORD=(WORD), POS=TAG, TREE=TREE, CHART=CHART) traindata = Dataset(transform, args.train) WORD.build(traindata, 2, None) CHART.build(traindata) TAG.build(traindata) args.encoder = 'lstm' args.n_words = WORD.vocab.n_init args.n_labels = len(CHART.vocab)
def setUp(self): self.fields = {'source': Field(), 'target': Field()} self.examples = {'source': [1, 2, 3], 'target': [1, 2, 3]} self.dataset = Dataset(self.examples, self.fields)
def setUp(self): self.field = Field()
class User: _fields = dict( email=Field(), name=Field, )