def test_setitem(self, empty_profile, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
# empty table
table = itsdb.Table(fields)
with pytest.raises(IndexError):
table[0] = (10, 'The bird chirps.')
# detached
table.append((10, 'The bird chirps.'))
table[0] = (10, 'The bird chirped.')
assert len(table) == 1
assert table[0] == (10, 'The bird chirped.')
# attached
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert table[0] == (0, 'The dog barks.')
table[0] = (0, 'The dog barked.')
assert table[0] == (0, 'The dog barked.')
# slice
table = itsdb.Table(fields)
table[:] = [(0, 'The whale sings.')]
assert len(table) == 1
assert table[0] == (0, 'The whale sings.')
table[0:5] = [(0, 'The whale sang.'), (1, 'The bear growls.')]
assert len(table) == 2
assert table[-1] == (1, 'The bear growls.')
table[-1:] = []
assert len(table) == 1
assert table[-1] == (0, 'The whale sang.')
def test_init(self, empty_item_table):
dir = empty_item_table.dir
fields = empty_item_table.fields
with pytest.raises(TypeError):
itsdb.Table()
with pytest.raises(TypeError):
itsdb.Table(dir)
with pytest.raises(TypeError):
itsdb.Table(dir, 'item')
# empty table
table = itsdb.Table(dir, 'item', fields)
assert len(table) == 0
assert table.name == 'item'
assert table.fields == fields
assert table.dir == dir
assert table.encoding == 'utf-8'
def test_write(self, tmpdir):
fields = itsdb.Relations.from_string(_simple_relations)['item']
table = itsdb.Table(fields, records=[(10, 'Birds chirp.')])
path = tmpdir.join('item')
with pytest.raises(itsdb.ItsdbError):
table.write() # cannot write to detached table without *path*
table.write(path=str(path)) # can write with *path*
assert path.check()
assert open(str(path)).read() == '10@Birds chirp.\n'
# writing a new record to the path overwrites
table.write(records=[(20, 'Cats meow.')], path=str(path))
assert open(str(path)).read() == '20@Cats meow.\n'
# unless append=True
table.write(records=[(30, 'Cows moo.')], path=str(path), append=True)
assert open(str(path)).read() == '20@Cats meow.\n30@Cows moo.\n'
# gzip compresses and deletes any uncompressed version
table.write(path=str(path), gzip=True)
assert not path.check()
assert tmpdir.join('item.gz').check()
# attached tables may be written without *path*
table = itsdb.Table.from_file(str(path), fields=fields)
table.write(records=[(10, 'Birds chirp.')], gzip=False)
assert path.check()
assert not tmpdir.join('item.gz').check()
assert table.path == str(path)
assert open(str(path)).read() == '10@Birds chirp.\n'
# ensure path is updated when gzip option is used
table.write(gzip=True)
assert table.path == str(path) + '.gz'
table.write(gzip=False)
assert table.path == str(path)
def test_list_changes(self, empty_profile, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
table = itsdb.Table(fields)
# detached tables do not track changes
with pytest.raises(itsdb.ItsdbError):
table.list_changes()
table.append((10, 'Dogs bark.'))
with pytest.raises(itsdb.ItsdbError):
table.list_changes()
# attching to a new file makes uncommitted records into changes
table.attach(os.path.join(empty_profile, 'item'))
assert len(table.list_changes()) == 1
# writing an attached table clears changes
table.write()
assert table.list_changes() == []
# attaching newly to a testsuite has no changes
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert table.list_changes() == []
# appending introduces a change
table.append((10, 'Dogs bark.'))
assert table.list_changes() == [(1, table[1])]
# modifying a record introduces a change
table[0]['i-input'] = 'The dog had barked.'
assert table.list_changes() == [(0, table[0]), (1, table[1])]
# detaching makes changes untrackable again
table.detach()
with pytest.raises(itsdb.ItsdbError):
table.list_changes()
def test_init(self):
fields = itsdb.Relations.from_string(_simple_relations)['item']
with pytest.raises(TypeError):
itsdb.Table() # no relations
# empty table
table = itsdb.Table(fields=fields)
assert len(table) == 0
assert table.name == 'item'
assert table.fields == fields
assert table.path is None
assert table.encoding is None
# table with a record
table = itsdb.Table(fields=fields, records=[(10, 'Birds chirp.')])
assert len(table) == 1
assert table[0]['i-id'] == 10
assert table[0]['i-input'] == 'Birds chirp.'
def _transitive_join(tab1, tab2, ts, how):
if tab1 is None:
table = itsdb.Table(tab2.fields, list(tab2))
else:
table = tab1
# the tables may not be directly joinable but could be
# joinable transitively via a 'path' of table joins
path = ts.relations.path(tab1.name, tab2.name)
for intervening, pivot in path:
table = itsdb.join(table, ts[intervening], on=pivot, how=how)
return table
def test_detach(self, tmpdir, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
empty_fn = tmpdir.mkdir('tmp').join('item')
item_fn = os.path.join(single_item_skeleton, 'item')
# detach unchanged table from empty file
table = itsdb.Table(fields)
table.attach(str(empty_fn))
assert len(table) == 0
table.write()
assert open(str(empty_fn)).read() == ''
table.detach()
assert len(table) == 0
# detach changed table from empty file
table = itsdb.Table(fields)
table.attach(str(empty_fn))
assert len(table) == 0
table.append((10, 'Birds chirp.'))
assert len(table) == 1
assert open(str(empty_fn)).read() == ''
table.detach()
assert len(table) == 1
assert open(str(empty_fn)).read() == ''
# detach unchanged table from non-empty file
table = itsdb.Table(fields)
table.attach(item_fn)
assert len(table) == 1
table.detach()
assert len(table) == 1
# detach changed table from non-empty file
table = itsdb.Table(fields)
table.attach(item_fn)
assert len(table) == 1
assert table[0]['i-input'] == 'The dog barks.'
table[0]['i-input'] = 'The bird chirps.'
table.append((20, 'Cats meow.'))
table.detach()
assert len(table) == 2
assert table[0]['i-input'] == 'The bird chirps.'
assert open(item_fn).read() == '0@The dog barks.'
def test_is_attached(self, single_item_skeleton,
gzipped_single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
table = itsdb.Table(fields)
assert table.is_attached() == False
# explicit attachment
table.attach(os.path.join(single_item_skeleton, 'item'))
assert table.is_attached() == True
# explicit detachment
table.detach()
assert table.is_attached() == False
# from_file attachment
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert table.is_attached() == True
# from_file attachment with gzipped file
table = itsdb.Table.from_file(
os.path.join(gzipped_single_item_skeleton, 'item'))
assert table.is_attached() == True
# writing does not attach
table = itsdb.Table(fields, [(10, 'Birds chirp.')])
table.write(path=os.path.join(single_item_skeleton, 'item'))
assert table.is_attached() == False
def test_select(self, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
# empty table
table = itsdb.Table(fields)
assert list(table.select('item:i-id')) == []
# detached
table.append((10, 'The bird chirps.'))
assert list(table.select('item:i-id')) == [[10]]
# attached and synced
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert list(table.select('item:i-id')) == [[0]]
# attached with unsynced records
table.append((1, 'The bear growls.'))
assert list(table.select('item:i-id')) == [[0], [1]]
def test_append(self, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
# on bare table
table = itsdb.Table(fields)
assert len(table) == 0
table.append((10, 'The dog barks.'))
assert len(table) == 1
assert table[-1]['i-input'] == 'The dog barks.'
table.append((20, 'The cat meows.'))
assert len(table) == 2
assert table[-1]['i-input'] == 'The cat meows.'
# on attached table
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert len(table) == 1
table.append((20, 'The bird chirps.'))
assert len(table) == 2
assert table[-1]['i-input'] == 'The bird chirps.'
def test_getitem(self, empty_profile, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
# empty table
table = itsdb.Table(fields)
with pytest.raises(IndexError):
table[0]
# detached
table.append((10, 'The bird chirps.'))
assert table[0] == (10, 'The bird chirps.')
assert table[-1] == (10, 'The bird chirps.')
# attached and synced
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert table[0] == (0, 'The dog barks.')
# attached with unsynced records
table.append((1, 'The bear growls.'))
assert table[-1] == (1, 'The bear growls.')
# slice
assert table[:] == [(0, 'The dog barks.'), (1, 'The bear growls.')]
assert table[0:1] == [(0, 'The dog barks.')]
assert table[::2] == [(0, 'The dog barks.')]
assert table[::-1] == [(1, 'The bear growls.'), (0, 'The dog barks.')]
def test_extend(self, single_item_skeleton):
fields = itsdb.Relations.from_string(_simple_relations)['item']
# on bare table
table = itsdb.Table(fields)
assert len(table) == 0
table.extend([(10, 'The dog barks.')])
assert len(table) == 1
assert table[-1]['i-input'] == 'The dog barks.'
def record_generator():
yield (20, 'The cat meows.')
yield (20, 'The horse whinnies.')
yield (20, 'The elephant trumpets.')
table.extend(record_generator())
assert len(table) == 4
assert table[-1]['i-input'] == 'The elephant trumpets.'
# on attached table
table = itsdb.Table.from_file(
os.path.join(single_item_skeleton, 'item'))
assert len(table) == 1
table.extend(record_generator())
assert len(table) == 4
assert table[-1]['i-input'] == 'The elephant trumpets.'
def test_attach(self, empty_profile):
fields = itsdb.Relations.from_string(_simple_relations)['item']
item_fn = os.path.join(empty_profile, 'item')
enc = 'utf-8'
# attach empty table to empty file
table = itsdb.Table(fields)
assert not os.path.exists(item_fn)
table.attach(item_fn)
assert os.path.exists(item_fn) # attaching creates the file
table.write()
assert open(item_fn).read() == ''
# attach already attached table
with pytest.raises(itsdb.ItsdbError):
table.attach(item_fn)
os.unlink(item_fn)
# attach non-empty table to empty file
table = itsdb.Table(fields, records=[(10, 'Birds chirp.')])
table.attach(item_fn)
assert open(item_fn).read() == '' # nothing written yet
table.write()
assert open(item_fn).read() == '10@Birds chirp.\n'
# attach empty table to non-empty file
table = itsdb.Table(fields)
assert len(table) == 0
table.attach(item_fn)
assert len(table) == 1
# attaching with .gz filename
table2 = itsdb.Table(fields)
table2.attach(item_fn + '.gz')
assert len(table2) == 1
assert table2.path == item_fn
# attaching to gzipped tables
table.write(gzip=True)
table2 = itsdb.Table(fields)
table2.attach(table.path)
assert len(table2) == 1
assert table2.path == item_fn + '.gz'
table.write(gzip=False) # just reset for the next test
# attach non-empty table to non-empty file
table = itsdb.Table(fields, records=[(20, 'Wolves howl.')])
with pytest.raises(itsdb.ItsdbError):
table.attach(item_fn)
assert open(item_fn).read() == '10@Birds chirp.\n'
def empty_item_table(empty_testsuite):
fields = tsdb.read_schema(empty_testsuite)['item']
table = itsdb.Table(empty_testsuite, 'item', fields)
return table
def single_item_table(single_item_skeleton):
fields = tsdb.read_schema(single_item_skeleton)['item']
table = itsdb.Table(single_item_skeleton, 'item', fields)
return table
def process(grammar,
testsuite,
source=None,
select=None,
generate=False,
transfer=False,
options=None,
all_items=False,
result_id=None,
gzip=False):
"""
Process (e.g., parse) a [incr tsdb()] profile.
Results are written to directly to *testsuite*.
If *select* is `None`, the defaults depend on the task:
========== =========================
Task Default value of *select*
========== =========================
Parsing `item:i-input`
Transfer `result:mrs`
Generation `result:mrs`
========== =========================
Args:
grammar (str): path to a compiled grammar image
testsuite (str): path to a [incr tsdb()] testsuite where data
will be read from (see *source*) and written to
source (str): path to a [incr tsdb()] testsuite; if `None`,
*testsuite* is used as the source of data
select (str): TSQL query for selecting processor inputs
(default depends on the processor type)
generate (bool): if `True`, generate instead of parse
(default: `False`)
transfer (bool): if `True`, transfer instead of parse
(default: `False`)
options (list): list of ACE command-line options to use when
invoking the ACE subprocess; unsupported options will
give an error message
all_items (bool): if `True`, don't exclude ignored items
(those with `i-wf==2`) when parsing
result_id (int): if given, only keep items with the specified
`result-id`
gzip (bool): if `True`, non-empty tables will be compressed
with gzip
"""
from delphin.interfaces import ace
if generate and transfer:
raise ValueError("'generate' is incompatible with 'transfer'")
if source is None:
source = testsuite
if select is None:
select = 'result:mrs' if (generate or transfer) else 'item:i-input'
if generate:
processor = ace.AceGenerator
elif transfer:
processor = ace.AceTransferer
else:
if not all_items:
select += ' where i-wf != 2'
processor = ace.AceParser
if result_id is not None:
select += ' where result-id == {}'.format(result_id)
source = itsdb.TestSuite(source)
target = itsdb.TestSuite(testsuite)
column, tablename, condition = _interpret_selection(select, source)
table = itsdb.Table(
source[tablename].fields,
tsql.select('* from {} {}'.format(tablename, condition),
source,
cast=False))
with processor(grammar, cmdargs=options) as cpu:
target.process(cpu, ':' + column, source=table, gzip=gzip)