class TestField(unittest.TestCase):
def setUp(self):
self.field = Field()
def test_numericalize_and_preprocess(self):
example = 1
result = self.field.numericalize(example)
self.assertEqual(example, result)
result = self.field.preprocess(example)
self.assertEqual(example, result)
def test_from_json(self):
json_obj = [{'source': [1, 2, 3], 'target': [4, 5, 6]},
{'source': [1, 2, 3], 'target': [4, 5, 6]}]
file_path = '/path/to/json_file'
import json # noqa: F401
open_patcher = patch('data.open')
json_load_patcher = patch('json.load', return_value=json_obj)
open_mock = open_patcher.start()
json_load_mock = json_load_patcher.start()
source = Field()
target = Field()
examples = Loader.from_json(file_path, {'source': source, 'target': target})
open_mock.assert_called_once_with(file_path)
json_load_mock.assert_called_once_with(open_mock(file_path).__enter__())
self.assertEqual(len(examples['source']), 2)
self.assertEqual(len(examples['target']), 2)
self.assertListEqual(examples['source'].data, [[1, 2, 3], [1, 2, 3]])
self.assertListEqual(examples['target'].data, [[4, 5, 6], [4, 5, 6]])
open_patcher.stop()
json_load_patcher.stop()
args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
args.mbr = True
args.delete = {
'TOP', 'S1', '-NONE-', ',', ':', '``', "''", '.', '?', '!', ''
}
args.equal = {'ADVP': 'PRT'}
args.train = os.path.join(args.data, 'train.pid')
args.dev = os.path.join(args.data, 'dev.pid')
args.test = os.path.join(args.data, 'test.pid')
print("Building the fields")
WORD = Field('words',
pad='<pad>',
unk='<unk>',
bos='<bos>',
eos='<eos>',
lower=True)
TAG = Field('tags', bos="<bos>", eos='<eos>')
TREE = RawField('trees')
global CHART
CHART = ChartField('charts')
transform = Tree(WORD=(WORD), POS=TAG, TREE=TREE, CHART=CHART)
traindata = Dataset(transform, args.train)
WORD.build(traindata, 2, None)
CHART.build(traindata)
TAG.build(traindata)
args.encoder = 'lstm'
args.n_words = WORD.vocab.n_init
args.n_labels = len(CHART.vocab)
def setUp(self):
self.fields = {'source': Field(), 'target': Field()}
self.examples = {'source': [1, 2, 3], 'target': [1, 2, 3]}
self.dataset = Dataset(self.examples, self.fields)
def setUp(self):
self.field = Field()
class User:
_fields = dict(
email=Field(),
name=Field,
)